% Copyright (c) 2012, Loïc Grobol <loic.grobol@gmail.com>
%
% Permission is granted to Do What The Fuck You Want To
% with this document under the terms of the WTF Public License, Version 2
% as published by Sam Hocevar.
%
% See http://sam.zoy.org/wtfpl/COPYING for more details.


\RequirePackage[l2tabu, orthodox]{nag} % Warn for obsolete packages and common mistakes
%%%% Document type %%%%
\documentclass[a4paper]{memoir}
	\chapterstyle{madsen}

%%%% Packages %%%%%
\usepackage[utf8x]{luainputenc} % Encoding for lua
\usepackage{luatextra} % Wrapper for luatex goodies
\usepackage{fontspec} % To use Opentype

\usepackage[british]{babel} % British typography

\usepackage{lmodern} % Latin Modern font

\usepackage{tikz} % Graphicz

\usepackage[autostyle=true]{csquotes} % Nice quotations
	\newcommand*{\openquote}{\tikz[remember picture,overlay,xshift=-15pt,text=gray!40]
     \node (OQ) {\Huge\bfseries\upshape“};\kern0pt}
	\newcommand*{\closequote}{\tikz[remember picture,overlay,xshift=15pt,text=gray!40]
     \node (CQ) {\Huge\bfseries\upshape”};}

	% Cite in italics
	\renewcommand{\mktextquote}[6]{%
		#1%
		\itshape%
		#2#4#3#6#5%
	}
	\renewcommand{\mkblockquote}[4]{%
		\itshape%
		#1#2#4#3%
	}
	\renewcommand{\mkbegdispquote}[2]{%
		\openquote%
		\itshape%
	}
	\renewcommand{\mkenddispquote}[2]{%
		\hfill
		\closequote\upshape\newline%
		\hfill #1#2%
	}
\usepackage[backend=biber,style=alphabetic,backref=true,block=ragged]{biblatex} % Bibliography, suggest newline between blocks

\usepackage{acronym}
	% Put the acronyms in the index
	\let\oldac\ac
\renewcommand*{\ac}[1]{\oldac{#1}\index{\acf{#1}}}

\usepackage{booktabs}	% beautiful tables
\usepackage{fixltx2e}	% sub/super-scripts in text mode and misc.
\usepackage{enumitem}	% beautiful enums
\usepackage{fancyvrb}	% beautiful verbatims
\usepackage{numprint}	% beautiful numbers
\usepackage{xspace}	% clean spaces
\usepackage[expansion=false]{microtype}	% fine typesetting
% \usepackage{subcaption}
% 	\setlength{\belowcaptionskip}{1em}

\usepackage{makeidx} % Index
\makeindex % Generate the index
\usepackage{tocbibind} % Add Toc in toc

% Empty style for pages before chapters
\newcommand{\clearemptydoublepage}{\newpage{\pagestyle{plain}\cleardoublepage}}

%%%% Troublesome packages %%%%
\usepackage[backref,pagebackref=false,hyperindex=true,bookmarks=true]{hyperref} % Liens hypertexte
\hypersetup{% Config d'hyperref, peut-être superflu
  pdfencoding=auto,	% détection d'encodage
  colorlinks=true,	% colorise les liens
  breaklinks=true,	% permet le retour \`a la ligne dans les liens trop longs
  urlcolor= blue,	% couleur des hyperliens
  linkcolor= red,	% couleur des liens internes
  bookmarksopen=true,	% si les signets Acrobat sont créés, les afficher complètement.
  pdftitle={CTESLO documentation},	% informations apparaissant dans
  pdfauthor={Loïc Grobol <loic.grobol@gmail.com>},	% dans les informations du document
  pdfsubject={}	% sous Acrobat.
}
\usepackage{cleveref}	% Références
\crefformat{footnote}{#2\footnotemark[#1]#3} % Hack, allow multiple refrences to a footnote « some text\footnote{\label{first}footnote} some text\cref{first} », see http://tex.stackexchange.com/q/10102/8547

%%%% Déclaration des macros %%%%

% Conventions typo %
% Pour ajouter un peu de marquage sémantique
\newcommand{\etr}[1]{\emph{#1}}   % Pour les locutions d'origine étrangère.

%%% Bibliographie %%%
\addbibresource{biblio.bib} % Ajout d'un fichier de bibliographie
\nocite{*} % Cite toutes les entrées bibliographiques, même celles non-référencées

%%%% Headers/Footers %%%%
% \usepackage{fancyhdr}
% \fancyhf{}
% \pagestyle{fancy}
% \renewcommand{\chaptermark}[1]{\markboth{#1}{}}
% \renewcommand{\headrulewidth}{0pt}
% \fancyhead[LO]{CTESLO documentation}
% \fancyhead[RE]{\rightmark}
% \fancyfoot[C]{\thepage}
% \fancyfoot[R]{}

%%%% Titre %%%%
\title{CTESLO documentation\\(Draft \today)}
\author{Loïc Grobol\\
		Faculté de lettres, langues et sciences humaines\\
		Université d'Orléans}
\date{\today}

%%% Document-specific %%%
\newcommand{\cteslo}{\texttt{cteslo}}
\newcommand{\eg}{\emph{e.g.}}
\acrodef{TSV}{Tab-Separated Values}
\acrodef{sem}{Segmenteur-Étiqueteur Markovien}
\acrodef{pos}{Part Of Speech}
\acrodef{Lefff}{LExique des Formes Fléchies du Français}
\acrodef{Alexina}{Atelier pour les LEXiques INformatiques et leur Acquisition}
\acrodef{CTESLO}{Crf-Tagging ESLO}
%%%% Document %%%%

\begin{document}
\frontmatter
\begin{titlingpage}
	\begingroup% MS Thesis
	\vspace*{0.18\textheight}
	\centering
	\vfill
	{\fontsize{60pt}{30pt}\selectfont\sffamily CTESLO}\\[2\baselineskip]
	{\HUGE\sffamily Segmentation and Morphosyntactical Tagging Toolset}\\[4\baselineskip]
	\vfill
	{\Huge\bfseries Documentation and User's Guide}\par
	{\Huge Draft \today}\par
	\vfill
	{\LARGE Loïc Grobol\\Université d'Orléans\\\texttt{loic.grobol@gmail.com}}\par
	\endgroup
\end{titlingpage}

\cleardoublepage
\tableofcontents

\clearemptydoublepage
\chapter*{Thanks}
\addcontentsline{toc}{chapter}{Thanks}

\mainmatter
\chapter{Background}
\section{Morphosyntactic tagging}
\section{ESLO}
\section{Machine learning}
\section{Motivation}


\chapter{History}
\section{Versions}
A detailed timeline is available at \url{https://launchpad.net/cteslo/+series}.
\section{Version 0.7 “plain”}
\paragraph{Description}
First usable version with stable UI and tagging features. Released on 2011-10-29.
\paragraph{Changelog}
N/A
\paragraph{Notes}

\section{Models}
The current built-in models have been trained on a small subset of the ESLO corpus using Wapiti \parencite{lavergne2010practical} with the patterns sets available in \path{taggers/wapiti/models/trained_with/}.
\section{Corpus}
At the beginning of the project, a small subset of ESLO has been manually tagged. It consisted of five \ac{TSV} files that looked like
\begin{verbatim}
oui      oui      ADV ADV     ADV
en_effet en_effet ADV ADV     ADV
on       on       P   P3MS    P3MSPERSUJ
peut     pouvoir  V   V3SINDP V3SINDP
\end{verbatim}

There were some issues with this original corpus.

\subsection{Loss of diacritics}
For yet unknown reasons, approximately 75 percent of the corpus had lost all characters with diacritics, \eg{} \texttt{Orléans} had been converted to \texttt{Orlans}. This part of the corpus was thus not usable. Especially since all occurrences of \texttt{à} had been skipped, leading to bizarre sequences, \eg
\begin{Verbatim}[samepage=true]
qu'est-ce_qui PIINT
vous          P2PPERCOMPL
a             V3SINDPAUX
amen          VMSPP
              PREP
vivre         VINF
ic            ADV
\end{Verbatim}

The issue has been dealt with, using the original transcriptions to bring back these characters. The main downside of this process is that the original typos that had been corrected in manually tagged files have also been brought back.

\subsection{Various inconsistencies}
During the aforementioned process of aligning the original transcription with manual labels, some other issues were detected with the labelled version, mainly recurrent replacements

\begin{tabular}{cc}
  \toprule
    Original	&	Replaced by\\
  \midrule
    \texttt{ah bon ?}	&	\texttt{n'a pas}\\
    \texttt{on a pas}	&	\texttt{on n'a pas}\\
    \texttt{y'a}	&	\texttt{y a}\\
  \bottomrule
\end{tabular}

In addition to that, some lines were not well-formed \ac{TSV} lines, tabs character having been replaced with spaces.

This issues had to be dealt with manually, mainly with \texttt{sed} and a text editor's find-and-replace.

\subsection{Encoding}
While this is technically not an issue, the encoding of the files in \texttt{cp1252} had to be changed to \texttt{utf-8}.

\section{Planned features}
Blueprints for planned features are available at \url{https://blueprints.launchpad.net/cteslo}

\chapter{Using CTESLO}
\section{Basic notions}
\section{Features}
CTESLO main tool, \texttt{cteslo.py}, provides the following functions. The other tools included in CTESLO are detailed in \cref{builtins}. Short usages notes are available using \texttt{cteslo.py --help}.
\subsection{Tagger}\label{usingtagger}
The main purpose of CTESLO is the morphosyntactic tagging of language corpora. This is achieved with the command \texttt{tag}, that accept the following options
\begin{description}
  \item[\texttt{--in-stream IN\_STREAM, -i IN\_STREAM}] The text file or stream to be tagged. The encoding \emph{must} be \texttt{utf-8}

  \item[\texttt{--out-stream OUT\_STREAM, -o OUT\_STREAM}] The file or stream where the tagged file will be outputted

  \item[\texttt{--format FORMAT, -f FORMAT}] The format of the input file. Must be an available format as described in \cref{formats}
\end{description}

\subsection{Training}
\subsection{Evaluation}
\section{Performances}

\chapter{Structure}
\section{Main routine}
The goal of CTESLO is to generate a file that look like
\begin{verbatim}
oui      ADV
en_effet ADV
on       P3MSPERSUJ
peut     V3SINDP
\end{verbatim}
With one word and its label on each line from an input like
\begin{verbatim}
oui en effet on peut
\end{verbatim}
This last format will be called \emph{plain text}\index{Plain text} and the first \emph{tagged tsv}\index{Tagged tsv} throughout the rest of this document\label{texts}

\subsection{Segmentation}\label{processseg}
The first thing we can note is this line
\begin{verbatim}
en_effet ADV
\end{verbatim}
the underscore \texttt{\_} means that this is one entity made of two words, a \emph{multi-word lexeme}\index{multi-word lexeme}. In order to tag the input accurately, we have to segment it in lexemes, rather than in words.

As for the actual tagging, we have two options
\begin{itemize}
  \item Rule-based segmentation
  \item Machine-learned segmentation
\end{itemize}
the development of a segmentation engine was not planned \emph{a priori}, so instead of starting the tedious development of a rule-based one, the solution was to develop tools that could handle both morphosyntactic tagging and segmentation. There was actually not much things to do, since segmentation can be reduced to tagging with two labels
\begin{description}
  \item[B -- Begin] labels the first word of a lexeme
  \item[I -- Inside] labels the following words of a multi-word lexeme
\end{description}
and a some simple text file manipulations.

The first thing to do is then to segment the input as much as possible, to get atomic units\index{atomic} that are strict subsets of lexemes. So our original input is first segmented to
\begin{verbatim}
oui
en
effet
on
peut
\end{verbatim}
To which we add the data from the external resources\footnote{See \cref{chap|resources}} --- here for the sake of clarity only \texttt{sem} labels --- to get
\begin{verbatim}
oui   ADV
en    P
effet NC
on    CLS
peut  V
\end{verbatim}
That is then tagged by Wapiti into
\begin{verbatim}
oui   ADV  B
en    P    B
effet NC   I
on    CLS  B
peut  V    B
\end{verbatim}
Finally, we concatenate this into
\begin{verbatim}
oui      ADV
en_effet P_NC
effet    NC
on       CLS
peut     V
\end{verbatim}

Voilà! We have a CRF-segmented corpus.

\subsection{Tagging}
Now for the next and most wanted step, we want to add morphosyntactic labels to the last result. Well, it is not that harder, for all the formatting and segmenting is already done. We just have to run Wapiti on the result of the segmentation step with the appropriate model, and we get
\begin{verbatim}
oui      ADV   ADV
en_effet P_NC  ADV
on       CLS   P3MSPERSUJ
peut     V     V3SINDP
\end{verbatim}
The only thing left is the cleaning into
\begin{verbatim}
oui      ADV
en_effet ADV
on       P3MSPERSUJ
peut     V3SINDP
\end{verbatim}

The whole process is summed up in \cref{mainprocess}

\begin{figure}
	\centering
	\includegraphics[width=0.8\textwidth]{SegTag}
	\caption{Whole segmentation and tagging process}
	\label{mainprocess}
\end{figure}

\section{Formats}\label{formats}
The segmentation and morphosyntactic tagging of plain text files is not always the whole task. The need is often the more complicated tagging of speeches enclosed inside complex files --- the XML-based \texttt{trs}\index{trs} format of Transcriber\index{Transcriber} for instance.

\subsection{Principle}
Since we have already done the effort of building a complete tagger for plain text files, the most sensible way to perform this was to
\begin{enumerate}
  \item Translate said complex files into plain text
  \item Segment and tag them using the regular routine
  \item Merge the result with the original file to get both its formatting, its data and our tags
\end{enumerate}
So the process described by \cref{mainprocess} becomes the one of \cref{formatprocess}, the format of the input file being specified in the call to \texttt{cteslo} as described in \cref{usingtagger}.
\begin{figure}
	\centering
	\includegraphics[width=0.8\textwidth]{SegTagFor}
	\caption{Tagging process with formatting}
	\label{formatprocess}
\end{figure}

\subsection{Implementation}\label{formatsimpl}
For the sake of adaptability, we choose to use a plugin structure to handle formats : every format has its own set of scripts that the main engine merely runs on input and output files to translate them to plain text and from \ac{TSV}. These plugins have to met the following requirements in order to be usable by CTESLO.

\begin{itemize}
  \item A format plugin is a sub-directory of \path{formats/}
  \item The name of the format is the name of said sub-directory
  \item The plugin must present an \path{__init__.py} valid Python script,
  \item The \path{__init__.py} must define a \texttt{NAME} value that contains the format's name
\end{itemize}

The \path{__init__.py}  can define other values that link to utility binaries. So far CTESLO uses the following utilities

\begin{description}
  \item[\texttt{TEXTIFIER}] Used to translate the original file into a plain text\footnote{\label{fttexttools}see \cref{texts}} file
  \item[\texttt{FORMATIFIER}] Used to translate the tagged tsv\cref{fttexttools} into the original format
\end{description}

These binaries must recognize the following parameters
\begin{description}
  \item[\texttt{-i}] To specify the input file
  \item[\texttt{-o}] To specify the output file
\end{description}
plus, for \texttt{FORMATIFIER}
\begin{description}
  \item[\texttt{-s}] To specify the original input file
\end{description}
So a typical call to an utility should look like
\begin{verbatim}
formatify.py -i input_file -o output_file -s original_file
\end{verbatim}

\chapter{Resources}\label{chap|resources}
\section{Principle}
In order to tag more accurately, CTESLO uses some external resources in its Wapiti models. Following the line of thoughts of \cref{formatsimpl}, these are integrated in a plugin structure.

Each resource plugin is responsible for the translation of a plain text file into a tagged trs, exactly as CTESLO. All the result, atomically segmented if needed, are concatenated with the atomically segmented file as described in \cref{processseg}.
\subsection{Implementation}

\section{Built-in}\label{builtins}
\subsection{Resources}
\subsubsection{sem}\index{sem}
\emph{\ac{sem}} is a \ac{pos} tagger developed by Yohann Dupont under the direction of Isabelle Tellier. Trained on Paris7's French Treebank, it features a basic segmentation engine, too.

Due to a random typo and a lack of refactoring so far, sem is available under \path{external_tools/cem}. The original home page is available at \url{http://www.univ-orleans.fr/lifo/Members/Isabelle.Tellier/SEM.html}.

\subsubsection{lglex-lefff}
lglex-lefff is a compound utility, made of Benoît Sagot's \ac{Lefff} and LADL's Lexique-Grammaire.

\paragraph{Lefff}\index{Lefff}
The \ac{Lefff} is lexicon of French's flexed forms developed by Benoît Sagot in the frame of the \ac{Alexina} project. Its extensional version features more than $500000$ flexed forms. Its \ac{TSV} format is also quite convenient for data extraction

\paragraph{LGLex}\index{LGLex}
LGLex is a  syntactic lexicon developed by Elsa Tolonne and Matthieu Constant \parencite{ConTol09}. It integrates the data of Maurice Gross's Lexique-Grammaire du Français. \textcite{Tol11phd} describe the process of translating LGLex native format into the Lefff's format. Tolonne made available partial results of this translation process, namely the \emph{simple verbs and predicative nouns} table at \cite{tolonne2011lgleflefff}.

\paragraph{lglex-lefff}
lglex-lefff is the concatenation of aforementioned \ac{Lefff} and LGLex' Lefff dump data into a Python pickle and a script that uses this pickle to add data to atomic words. The script thus doesn't perform any kind of segmentation and merely use CTESLO atomic segmentation engine to get a tagged \ac{TSV} fi

Taking advantage of the free\footnote{As in free speech, of course.} licenses of both works, it uses the following data if available
\begin{itemize}
  \item Parts of speech
  \item Lemma
  \item Types
\end{itemize}

In case of homonyms, which happens quite often, it only concatenates the data of all homonyms. For instance, if \emph{laisse} has two homonyms which lemma are \emph{laisser} and \emph{laisse}, then each instance of \emph{laisse} will receive the lemma \emph{laisse, laisser}.

\section{Taggers}
\subsection{Wapiti}

\section{Formats}
\subsection{Plain text}\index{Plain text}

\subsection{Tagged tsv}\index{Tagged tsv}

\subsection{Tagged trs}\index{Tagged trs}
The Tagged trs format is a variation on Claude Barras' Transcriber \parencite{Transcriber-SpeechComm2000} native format trs\index{trs}. The only difference between the two format is that trs encloses speech as text childs of \texttt{<Turn>} nodes, while Tagged trs encloses them in additional \texttt{<w>}\index{<w>} nodes, the attribute of which specify the morphosyntactic labels coming from CTESLO tagging process. This syntax is essentially borrowed from the French Treebank\index{French Treebank} corpus. For instance here is a native trs
\begin{Verbatim}[gobble=1]
	<Trans>
	<Episode>
	<Section>
	<Turn startTime="1.317" endTime="3.593">
	<Sync time="1.317"/>
	euh
	<Sync time="2.455"/>
	quatre ans
	<Sync time="2.934"/>
	</Turn>
	[…]
	</Section>
	</Episode>
	</Trans>
\end{Verbatim}
and here the corresponding Tagged trs
\begin{Verbatim}[gobble=1]
	<Trans>
	<Episode>
	<Section>
	<Turn startTime="1.317" endTime="3.593">
	<Sync time="1.317"/>
	<w cat="MDEUH">euh</w>
	<Sync time="2.455"/>
	<w cat="ADJ" mph="NUM">quatre</w>
	<w cat="NC" mph="MP">ans</w>
	<Sync time="2.934"/>
	</Turn>
	[…]
	</Section>
	</Episode>
	</Trans>
\end{Verbatim}

\chapter{Examples}
Some common uses of \cteslo.
\paragraph{Tagging a plain text file}
\begin{verbatim}
  cteslo.py tag -i my_text_file -o the_tagged_file
\end{verbatim}

\paragraph{Tagging a Transcriber export text file}
\begin{verbatim}
cteslo.py tag -i my_exported_file -o the_tagged_file -f transcriber_export_fr
\end{verbatim}

\paragraph{Tagging a \texttt{trs} file}
\begin{verbatim}
cteslo.py tag -i my_trs_file -o the_tagged_file -f trs
\end{verbatim}

\paragraph{Making a training corpus}
\begin{verbatim}
cteslo.py make_train -i my_tagged_corpus -l segmentation_train -t tag_train
\end{verbatim}

\paragraph{Making a evaluation corpus}
To evaluate morphosyntactic tagging, including segmentation
\begin{verbatim}
cteslo.py make_eval -i tagged_corpus -o evaluation_corpus
\end{verbatim}
To evaluate only segmentation
\begin{verbatim}
cteslo.py make_eval -i tagged_corpus -o evaluation_corpus --nolabels
\end{verbatim}

\chapter{Get CTESLO}
\section{Requirements}
CTESLO is intended to be as portable and as simple of use as possible. No install is required: provided that all files are present and the folder structure is preserved, CTESLO will work on any operating system that meets the following requirements.

\subsection{Main engine}
\begin{itemize}
  \item Python 3.2 or higher
  \item A program loader that support shebang (\texttt{\#!})
  \item \texttt{env} at \texttt{/usr/bin/env}
\end{itemize}

\subsection{\texttt{sem}}
\begin{itemize}
  \item Python 2.7 or higher in Python 2 series, reachable by \texttt{/usr/bin/env python}
\end{itemize}

\subsection{Wapiti}
There are a pre-compiled binaries of Wapiti included in CTESLO that may or may not work in your operating system. If you are experiencing issues with Wapiti, please get the source from \url{http://wapiti.limsi.fr}, compile it for your system and replace the following files by the result
\begin{itemize}
  \item \texttt{taggers/wapiti/bin/wapiti}
  \item \texttt{external\_tools/cem/tagger/wapiti}
\end{itemize}


\section{Download}
CTESLO stable releases are available in Launchpad at
\url{https://launchpad.net/cteslo}. You can also get the most recent development version with Bazaar using \texttt{bzr branch lp:cteslo}.

\backmatter

\printindex

\printbibliography[heading=bibintoc]

\chapter*{Licence}
\addcontentsline{toc}{chapter}{Licence}
Copyright (c) 2012, Loïc Grobol <loic.grobol@gmail.com>

Permission is granted to Do What The Fuck You Want To
with this document under the terms of the WTF Public License, Version 2
as published by Sam Hocevar.

See \url{http://sam.zoy.org/wtfpl/COPYING} for more details.
\end{document}