#==============================================================================
# Copyright (C) 2011 by Loïc Grobol
#
# This file is part of CTESLO.
# 
#    CTESLO is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
# 
#    CTESLO is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
# 
#    You should have received a copy of the GNU General Public License
#    along with CTESLO.  If not, see <http://www.gnu.org/licenses/>.
#==============================================================================

"""Tokenization library for text data files

Created on 19 mai 2011
@author: Loïc Grobol <loic.grobol@gmail.com>

Note : this implementation uses dflib.abstractions to
parse and write text data files, for general considerations
on text data files, see the docstring of dflib.abstractions

This module is a library for processing text data
files lexeme wise. i.e. for tokenizing and merging
data lines.

The functionalities are basically split in two parts
    - A function that handles a single line and
perform the desired operation
    - A function that applies said single-line
function to each line of a file
"""
import re
import copy
import itertools
from .abstractions import DataLine
from . import abstractions as dfa
import functools

DEFAULT_DELIMITER = "_"
_delimiters_keep_string = r"['\-?!]"
_delimiters_del_string = r"\s|[_\n]"
_delimiters_string = "(" + _delimiters_keep_string + ")|" + _delimiters_del_string

DEFAULT_DELIMITERS = re.compile(_delimiters_string)

DEFAULT_BIO = "BI"

# Cloaking dirty hack for appending with list.insert()
# see tokenize()
DEFAULT_BIO_COLUMN = -1

DEFAULT_LEXEME_COLUMN = 0


#@functools.lru_cache(maxsize=None) # This is made in strem_tokenize
# to coperate with functools.partial
def tokenize(data,
             delimiters=DEFAULT_DELIMITERS,
             bio=DEFAULT_BIO, bio_column=DEFAULT_BIO_COLUMN,
             lexeme_column=DEFAULT_LEXEME_COLUMN,
             missing_characters_placeholder=None,
             custom_lexemes=None):
    """Tokenize a DataLine into a list of DataLines

    The returned DataLines have the columns and values of %data
    except for %lexeme_column which contains the tokens and an additional
    %bio_column (default last) which contains
        - For the first token the first element of %bio
        - For the last token the last element of %bio
        - For the other tokens the second element of %bio.

    If a %missing_characters_placeholder is specified, it will replace
    empty lexemes and non-final empty words.
    The tokenization behave like the one made by re.split, i.e.
    delimiters between parentheses will be given their own line
    and the others will be deleted.

    Since compound lexemes are often redundant, this is cached for
    speed's sake.
    """
    lexeme = data[lexeme_column]

    words = re.split(delimiters, lexeme)
    # Here some black magic : split keeps the
    # delimiters between parentheses in the regex
    # but keep None items for the other delimiters.
    # So this delete None items.
    words = [word for word in words if word is not None]

    # Preserve missing words by replacing them with
    # placeholder if placeholder is non-empty
    # /!\ UGLY AND EXPERIMENTAL
    if missing_characters_placeholder:
        # Empty lines (only missing characters) : word ==['']
        if len(words) == 1 and not words[0]:
            words = [missing_characters_placeholder]
        # Delimiters-only lines : words == ['', <DELIMITER>, '']
        elif (len(words) == 3 and re.match(delimiters, words[1])
             and not words[0] and not words[2]):
            pass
        # Other multi-words lexemes
        elif len(words) > 1:
            # To preserve '-ending lexemes, the last token won't be replaced
            last = words[-1]
            words = [missing_characters_placeholder if word == "" else word for word in words[:-1]]
            words.append(last)

    # Expand custom lexemes, ex (fr) aux -> à les
    if custom_lexemes:
        # This returns a list of list : necessary since custom_lexemes[words] is a list
        words = [custom_lexemes[word] if word in custom_lexemes else [word] for word in words]
        # That we flatten
        words = list(itertools.chain(*words))

    infos = _list_del(data, lexeme_column)

    # Give the info columns to every word, cleaning after-tokenization
    # empty strings
    tokenized = [_list_insert(infos, lexeme_column, word)
                    for word in words if word]

    # If bio have to be appended, let's do it!
    if bio:
        # Handle negative indices for %bio_column
        # override the unintuitive negative indices handling
        # of list.insert
        if bio_column < 0:
            bio_column = len(data) + bio_column + 1
        # First word is always B
        tokenized[0].insert(bio_column, bio[0])
        # If more than one word...
        if len(tokenized) > 1:
            # Last word is O or I
            tokenized[-1].insert(bio_column, bio[-1])
            if len(tokenized) > 2:
                # Inside words are I
                tokenized[1:-1] = [_list_insert(token, bio_column, bio[1])
                                    for token in tokenized[1:-1]]
    return tokenized

#@functools.lru_cache(maxsize=None) # This is made in strem_tokenize
# to cooperate with functools.partial
def lexemize(data1, data2,
             default_delimiter, delimiters,
             lexeme_column,
             bio, bio_column,
             custom_lexemes) -> "DataLine":
    """Merge two DataLines into one. Meant for same number of columns.

    In the result, columns are in the same order,
    %default_delimiter tokenizes data, and words only if none
    of the words is already a %delimiter.

    Delete %bio_column and check BIO tags' consistency if %bio.
    Default %bio is "BI", so set bio="" to skip BIO check.
    Default %bio_column is -1 (i.e. last column), set
    %bio_column="" to skip BIO column deletion.

    A custom column can be specified in %lexeme_column.
    %custom_lexemes is a string:string dict that specify custom
    lexemizations.

    Since compound lexemes are often redundant, this is cached for
    speed's sake.

    >>> DL = dfa.DataLine("parce ADJ B".split())
    >>> DM = dfa.DataLine("que CS I".split())
    >>> DN = lexemize(DL, DM, "_")
    >>> assert DN.data == ["parce_que", "ADJ_CS"]
    >>> DN = lexemize(DL, DM, "_", custom_lexemes={"parce_que":"pake"})
    >>> assert DN.data == ["pake", "ADJ_CS"]
    """
    # Degenerated case : first line is empty
    if not data1:
        if bio:
            return _list_del(data2, bio_column)
        return data2
    # second line is empty. Zerology ftw.
    if not data2:
        if bio:
            return _list_del(data1, bio_column)
        return data1

    # Keep the words for inclusion after merging the others columns
    word1 = data1[lexeme_column]
    word2 = data2[lexeme_column]

    # Delete them to process them in their own way
    # but do it in a safe deepcopy : don't modify original DataLists
    data1_ = _list_del(data1, lexeme_column)
    data2_ = _list_del(data2, lexeme_column)

    # Check BIO consistency and del BIO column
    if bio:
        # Second position should never be a B 
        if not data2_[bio_column] in (bio[1], bio[-1]):
            raise Exception("Inconsistent BIO tags : {0} + {1}".format(
                                    data1_[bio_column], data2_[bio_column]))
        # First position should always be a B
        if not data1_[bio_column] == bio[0]:
            raise Exception("Inconsistent BIO tags : {0} + {1}".format(
                                    data1_[bio_column], data2_[bio_column]))

        del data1_[bio_column]
        del data2_[bio_column]

    res_list = [default_delimiter.join((token1, token2)) if (token1 != token2) else token1 for token1, token2 in zip(data1_, data2_)]

    word = _tokens_cat(word1, word2, default_delimiter, delimiters)
    # Handle custom lexemes' case (EXPERIMENTAL)
    if word in custom_lexemes:
        word = custom_lexemes[word]

    res_list.insert(lexeme_column, word)

    return DataLine(res_list)

def stream_tokenize(stream_in, stream_out,
                    column_delimiter=dfa.DEFAULT_DELIMITER,
                    words_delimiters=DEFAULT_DELIMITERS,
                    bio=DEFAULT_BIO, bio_column=DEFAULT_BIO_COLUMN,
                    lexeme_column=DEFAULT_LEXEME_COLUMN,
                    missing_characters_placeholder=None,
                    custom_lexemes=None):
    """Tokenize a text data file.
    Tokenization relies on tokenize().
    """
    # Avoid mutable default value bug
    custom_lexemes = {} if custom_lexemes is None else custom_lexemes
    tokenizer = functools.partial(tokenize,
                                  delimiters=words_delimiters,
                                  bio=bio, bio_column=bio_column,
                                  lexeme_column=lexeme_column,
                                  missing_characters_placeholder=missing_characters_placeholder,
                                  custom_lexemes=_HDict(custom_lexemes))
    # LRU cache to speed up
    tokenizer = functools.lru_cache(maxsize=None)(tokenizer)

    with dfa.datafile_open(stream_in, mode="r", in_delimiter=column_delimiter) as s_in, \
         dfa.datafile_open(stream_out, mode="w", out_delimiter=column_delimiter) as s_out:
        for line in s_in:
            if line.is_blank:
                s_out.write(line)
            else:
                lines = tokenizer(line)
                for tokenized_line in lines:
                    s_out.write(tokenized_line)

def stream_lexemize(stream_in, stream_out,
                    column_delimiter=dfa.DEFAULT_DELIMITER,
                    default_words_delimiter=DEFAULT_DELIMITER,
                    words_delimiters=DEFAULT_DELIMITERS,
                    lexeme_column=0,
                    bio=DEFAULT_BIO, bio_column=DEFAULT_BIO_COLUMN,
                    custom_lexemes=None):
    """Lexemize a text data file.

    Lexemization relies on lexemize().
    """
    # Avoid mutable default value issue
    custom_lexemes = {} if custom_lexemes is None else custom_lexemes
    # Partials to avoid passing parameters each time
    # and improve readability
    lexemizer = functools.partial(lexemize,
                                  default_delimiter=default_words_delimiter,
                                  delimiters=words_delimiters,
                                  lexeme_column=lexeme_column,
                                  bio=bio, bio_column=bio_column,
                                  custom_lexemes=_HDict(custom_lexemes))
    # LRU cache to speed up
    lexemizer = functools.lru_cache(maxsize=None)(lexemizer)

    insert_bio_B = functools.partial(_consistent_insert, index=bio_column, value=bio[0])

    with dfa.datafile_open(stream_in, mode="r", in_delimiter=column_delimiter) as s_in, \
         dfa.datafile_open(stream_out, mode="w", out_delimiter=column_delimiter) as s_out:
        line_buffer = DataLine()
        # For each line
        #    If it is blank, write the current buffer then flush it and write a
        #  blank line
        #    If it is a "B", write the current buffer, flush it then append
        #  the new line
        #    If it is a "O" append the new line to the buffer, write the buffer,
        #  then flush it
        #    Else append the new line to the buffer
        # The buffer's BIO column is always "B", so before any out.write, slice it
        # to remove the BIO column.
        for line in s_in:
            if line.is_blank:
                if line_buffer:
                    s_out.write(_list_del(line_buffer, bio_column))
                s_out.write(line)
                line_buffer = DataLine()
            else:
                # There is "O"
                if len(bio) > 2:
                    # And this is a "O"
                    if line[bio_column] == bio[-1]:
                        line_buffer = lexemizer(line_buffer, line)
                        s_out.write(_list_del(line_buffer, bio_column))
                        line_buffer = DataLine()
                # It is a "B"
                elif line[bio_column] == bio[0]:
                    # New lexeme
                    if line_buffer:
                        s_out.write(_list_del(line_buffer, bio_column))
                    # Flush the buffer and get the new line
                    line_buffer = line
                else:
                    line_buffer = lexemizer(line_buffer, line)
                    # The buffer is always "B"
                    insert_bio_B(line_buffer)
        # at EOF, out the remaining buffer
        if line_buffer:
            del line_buffer[bio_column]
            s_out.write(line_buffer)


def _tokens_cat(token1, token2, default_delimiter, delimiters):
    """Concatenate two strings, without delimiter if one is already a delimiter,
    if the first one ends with a delimiter or if the second one begins with a
    delimiter."""

    # Handles empty strings
    if not (token1 and token2):
        return (token1 + token2)
    # Note st[::-1] is inverse order st (from start to end with stride -1)
    if (re.match(delimiters, token1[::-1]) or
        re.match(delimiters, token2)):
        delimiter = ""
    else:
        delimiter = default_delimiter

    return delimiter.join((token1, token2))

# Magic functions for list-comprehensioning
def _list_insert(l, index, value):
    """Return a copy of %l with %value inserted at %index.

    >>> l = [2, 7, 1, 3]
    >>> m = _list_insert(l, 2, "Ni!")
    >>> assert m == [2, 7, "Ni!", 1, 3]
    """
    result = copy.deepcopy(l)
    result.insert(index, value)
    return result

def _list_del(l, index):
    """Return a copy of %l with %index deleted.

    >>> l = [2, 7, "Ni!", 1, 3]
    >>> m = _list_del(l, 2)
    >>> assert m == [2, 7, 1, 3]
    """
    result = copy.deepcopy(l)
    del result[index]
    return result

def _consistent_insert(l, index, value):
    """Insert an item at a position such that
    a lookup at this position will return said item,
    even for negative indices.

    >>> l = [2,1]
    >>> _consistent_inster(l, -1, 3)
    >>> _consistent_insert(l, 1, 7)
    >>> assert l[-1] == 3
    >>> assert l[1] == 7
    """
    insert_index = (len(l) - index + 1) if index < 0 else index
    l.insert(insert_index, value)


class _HDict(dict):
    """Hashable dictionary class. For use with functools.lru_cache."""
    def __hash__(self):
        return hash(frozenset(self.items()))
