#!/usr/bin/python3
# -*- coding: utf-8 -*-

#==============================================================================
# Copyright (C) 2011 by Loïc Grobol
#
# This file is part of CTESLO.
# 
#    CTESLO is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
# 
#    CTESLO is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
# 
#    You should have received a copy of the GNU General Public License
#    along with CTESLO.  If not, see <http://www.gnu.org/licenses/>.
#==============================================================================

"""Text files columnizer

Created on 17 mai 2011
@author: Loïc Grobol <loic.grobol@gmail.com>

This is a all-in-one text file columnizer. Here's the explanation.
Your average corpus text file is something like that.

    What is your name?
    My name is Sir Launcelot of Camelot
    What is your quest?
    To seek the Holy Grail
    What is your favorite color?
    Blue
    
Which this columnize as

    What
    is
    your
    name
    ?
    
    My
    name
    [...]

See the point? It put each word in its own line and a blank line
whenever an actual EOL is encountered.

The nasty bits here are
    - What is a word delimiter?
    - What delimiter do I want to be kept

Here the regex pattern _delimiters_keep_string is
the delimiter that will be kept and _delimiters_del_string
the delimiter that will be discarded.

So with default values

    we French fellows out-wit you
    
will be columnized into

    we
    French
    fellows
    out
    -
    wit
    you
    
it can be applied to any stream, (e.g. stdin) or text file and
 redirected the same way.

A typical call would be

    format -i clean_transcriber_file -o columnized

By default input is stdin and output is stdout for easy piping.
So you can do

    format < clean_transcriber_file > columnized
    
format --help will also give you an usage guide.
"""
import sys
import argparse
import re

# These delimiters should be kept and given their own line
_delimiters_keep_string = r"(['\-])\s?"
# These won't be kept in the clean file.
_delimiters_del_string = r"\s|[_]"
# Note to self : the trick is to parentheses-ize the delimiters
# you want to keep.
# see http://docs.python.org/dev/py3k/library/re.html?highlight=re#re.split

_delimiters_keep_pattern = re.compile(_delimiters_keep_string)
_delimiters_del_pattern = re.compile(_delimiters_del_string)
_delimiters_pattern = re.compile("|".join((_delimiters_del_string, _delimiters_keep_string)))

def format_line(line):
    """Tokenize %line on _tokenizers_*, keeping _tokenizers_keep, deleting _tokenizers_del."""
    words = re.split(_delimiters_pattern, line)
    # Delete deleted delimiters leftovers, aka ""
    words = [word for word in words if word]
    lines = "\n".join(words)
    lines = lines + "\n\n"
    return lines

def format_stream(in_stream, out_stream):
    """Delete all same-line occurrences of _delete_pattern from
    %in_stream and write the output in %out_stream."""
    for line_raw in in_stream:
        line_formated = format_line(line_raw)
        out_stream.write(line_formated)

def main(argv=sys.argv[1:]):
    """Main function."""
    parser = argparse.ArgumentParser(description="Format clean text exports of Transcriber.")
    parser.add_argument("--in_stream", "-i",
                         type=argparse.FileType('r'),
                         default=sys.stdin,
                         help="Clean transcriber export file to be formated. Standard input by default.")
    parser.add_argument("--out_stream", "-o",
                         type=argparse.FileType('w'),
                         default=sys.stdout,
                         help="Formated file to be written. Standard output by default.")
    args = parser.parse_args(argv)
    if args.in_stream.isatty():
        parser.print_help()
    else:
        format_stream(args.in_stream, args.out_stream)

if __name__ == "__main__":
    sys.exit(main())
