/***************************************************************************
 *   Copyright (C) 2008 by S. MANKOWSKI / G. DE BURE support@mankowski.fr  *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>  *
 ***************************************************************************/
/** @file
 * This file is Skrooge plugin for for CSV import / export.
 *
 * @author Stephane MANKOWSKI / Guillaume DE BURE
 */
#include "skgimportplugincsv.h"
#include "skgtraces.h"
#include "skgservices.h"
#include "skgbankincludes.h"
#include "skgimportexportmanager.h"

#include <klocale.h>
#include <kfilterdev.h>
#include <ksavefile.h>
#include <QCryptographicHash>
#include <qfileinfo.h>

/**
 * This plugin factory.
 */
K_PLUGIN_FACTORY(SKGImportPluginCsvFactory, registerPlugin<SKGImportPluginCsv>();)
/**
 * This plugin export.
 */
K_EXPORT_PLUGIN(SKGImportPluginCsvFactory("skrooge_import_csv", "skrooge_import_csv"))

SKGImportPluginCsv::SKGImportPluginCsv(QObject* iImporter, const QVariantList& iArg)
    : SKGImportPlugin(iImporter),
      m_csvSeparator(';'), m_csvHeaderIndex(-1)
{
    SKGTRACEIN(10, "SKGImportPluginCsv::SKGImportPluginCsv");
    Q_UNUSED(iArg);

    m_parameters["mapping_date"] = "^date";
    m_parameters["mapping_account"] = "^account";
    m_parameters["mapping_number"] = "^number|^num?ro";
    m_parameters["mapping_mode"] = "^mode|^type";
    m_parameters["mapping_payee"] = "^payee|^tiers";
    m_parameters["mapping_comment"] = "^comment|^libell?|^d?tail|^info";
    m_parameters["mapping_status"] = "^status|^pointage";
    m_parameters["mapping_bookmarked"] = "^bookmarked";
    m_parameters["mapping_category"] = "^cat\\w*gor\\w*";
    m_parameters["mapping_amount"] = "^value|^amount|^valeur|^montant|^credit|^debit";
    m_parameters["mapping_quantity"] = "^quantity";
    m_parameters["mapping_unit"] = "^unit";
    m_parameters["mapping_sign"] = "^sign|^sens";
    m_parameters["mapping_debit"] = "^-|^debit|^withdrawal";
    m_parameters["mapping_idgroup"] = "^idgroup";
    m_parameters["mapping_idtransaction"] = "^idtransaction";
    m_parameters["mapping_property"] = "";

    m_parameters["automatic_search_header"] = 'Y';
    m_parameters["header_position"] = '1';
    m_parameters["automatic_search_columns"] = 'Y';
    m_parameters["columns_positions"] = "";

    m_parameters["mode_csv_unit"] = 'N';
}

SKGImportPluginCsv::~SKGImportPluginCsv()
{
}

void SKGImportPluginCsv::setImportParameters(const QMap< QString, QString >& iParameters)
{
    SKGImportPlugin::setImportParameters(iParameters);

    if (m_parameters["automatic_search_header"] == "N") {
        int header_position = SKGServices::stringToInt(m_parameters["header_position"]);
        setCSVHeaderIndex(header_position);
    }

    if (m_parameters["automatic_search_columns"] == "N") {
        QStringList columns_positions =  m_parameters["columns_positions"].split('|');
        setCSVMapping(&columns_positions);
    }
}

bool SKGImportPluginCsv::isImportPossible()
{
    SKGTRACEIN(10, "SKGImportPluginCsv::isImportPossible");
    return isExportPossible();
}

QStringList SKGImportPluginCsv::getCSVMappingFromLine(const QString& iLine)
{
    QStringList output;
    QString line = iLine.trimmed();

    //Split first line
    QStringList csvAttributes = SKGServices::splitCSVLine(line, ';', true, &m_csvSeparator);
    int nb = csvAttributes.count();
    for (int i = 0; i < nb; ++i) {
        QString att = csvAttributes[i].toLower();

        //Search if this csv column is mapping a std attribute
        QMapIterator<QString, QString> csvMapperIterator(m_parameters);
        bool found = false;
        while (!found && csvMapperIterator.hasNext()) {
            csvMapperIterator.next();

            QString key = csvMapperIterator.key();
            if (key.startsWith(QLatin1String("mapping_"))) {
                key = key.right(key.length() - 8);
                if (key != "debit" &&
                        key != "property" &&
                        !csvMapperIterator.value().isEmpty() &&
                        QRegExp(csvMapperIterator.value(), Qt::CaseInsensitive).indexIn(att) != -1 &&
                        (!output.contains(key) || key == "comment" || key == "category" || key == "amount")) {
                    output.push_back(key);
                    found = true;
                }
            }
        }

        //Search if this csv column must be added as a property
        if (!found &&
                !m_parameters["mapping_property"].isEmpty() &&
                QRegExp(m_parameters["mapping_property"], Qt::CaseInsensitive).indexIn(att) != -1 &&
                !output.contains(att)) {
            output.push_back(att);
            found = true;
        }

        if (!found) output.push_back("");      //To ignore this column
    }
    return output;
}

SKGError SKGImportPluginCsv::setCSVMapping(const QStringList* iCSVMapping)
{
    SKGError err;
    SKGTRACEINRC(10, "SKGImportPluginCsv::setCSVMapping", err);

    m_csvMapping.clear();

    if (iCSVMapping == NULL) {
        //Automatic build
        //Open file
        QFile file(m_importer->getFileName());
        if (!file.open(QIODevice::ReadOnly)) {
            err.setReturnCode(ERR_INVALIDARG);
            err.setMessage(i18nc("Error message",  "Open file '%1' failed", m_importer->getFileName()));
        } else {
            QTextStream stream(&file);
            if (!m_importer->getCodec().isEmpty()) stream.setCodec(m_importer->getCodec().toAscii().constData());

            //Ignore useless lines
            int headerIndex = getCSVHeaderIndex();
            for (int i = 1; i < headerIndex; ++i)
                stream.readLine();

            //Get mapping
            if (!stream.atEnd()) {
                m_csvMapping = getCSVMappingFromLine(stream.readLine());
            } else {
                err.setReturnCode(ERR_INVALIDARG);
            }

            //close file
            file.close();
        }
    } else {
        //Manual build
        m_csvMapping = *iCSVMapping;
    }

    if (!err) {
        //Check if mandatory attributes have been found
        if (!m_csvMapping.contains("date") || !m_csvMapping.contains("amount"))
            err = SKGError(ERR_FAIL, i18nc("Error message",  "Columns date and amount not found. Set import parameters in settings."));
    }

    return err;
}

QStringList SKGImportPluginCsv::getCSVMapping() const
{
    SKGTRACEIN(10, "SKGImportPluginCsv::getCSVMapping");
    return m_csvMapping;
}

SKGError SKGImportPluginCsv::setCSVHeaderIndex(int iIndex)
{
    SKGError err;
    SKGTRACEINRC(10, "SKGImportPluginCsv::setCSVHeaderIndex", err);

    if (iIndex == -1) {
        //Automatic build
        //Open file
        QFile file(m_importer->getFileName());
        if (!file.open(QIODevice::ReadOnly)) {
            err.setReturnCode(ERR_INVALIDARG);
            err.setMessage(i18nc("Error message",  "Open file '%1' failed", m_importer->getFileName()));
        } else {
            QTextStream stream(&file);
            if (!m_importer->getCodec().isEmpty()) stream.setCodec(m_importer->getCodec().toAscii().constData());

            int i = 1;
            m_csvHeaderIndex = -1;
            while (!stream.atEnd() && m_csvHeaderIndex == -1) {
                //Read line
                QStringList map = getCSVMappingFromLine(stream.readLine());
                if (map.contains("date") && map.contains("amount")) m_csvHeaderIndex = i;

                ++i;
            }

            //close file
            file.close();
        }
    } else {
        //Manual build
        m_csvHeaderIndex = iIndex;
    }

    return err;
}

int SKGImportPluginCsv::getCSVHeaderIndex()
{
    SKGTRACEIN(10, "SKGImportPluginCsv::getCSVHeaderIndex");
    if (m_csvHeaderIndex == -1) setCSVHeaderIndex(-1);
    return m_csvHeaderIndex;
}

QChar SKGImportPluginCsv::getCSVSeparator()
{
    SKGTRACEIN(10, "SKGImportPluginCsv::getCSVSeparator");
    return m_csvSeparator;
}

SKGError SKGImportPluginCsv::importFile()
{
    if (m_parameters["mode_csv_unit"] == "Y") return importCSVUnit();
    if (!m_importer) return SKGError(ERR_ABORT, i18nc("Error message", "Invalid parameters"));

    SKGError err;
    SKGTRACEINRC(2, "SKGImportPluginCsv::importFile", err);
    QString iFileName = m_importer->getFileName();
    SKGTRACEL(10) << "Input filename=" << iFileName << endl;

    //Begin transaction
    err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import %1 file", "CSV"), 3);
    if (!err) {
        //Initialize some variables
        QDateTime now = QDateTime::currentDateTime();
        QString postFix = SKGServices::dateToSqlString(now);

        //Default mapping
        if (getCSVMapping().count() == 0) {
            err = setCSVMapping(NULL);
            if (!err) err = m_importer->getDocument()->sendMessage(i18nc("An information message",  "Use automatic search of the columns"));
        }
        if (!err) err = m_importer->getDocument()->sendMessage(i18nc("An information message",  "Mapping used: %1", getCSVMapping().join("|")));

        //Step 1 done
        if (!err) err = m_importer->getDocument()->stepForward(1);

        //Open file
        if (!err) {
            QFile file(iFileName);
            if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
                err.setReturnCode(ERR_INVALIDARG);
                err.setMessage(i18nc("Error message",  "Open file '%1' failed", iFileName));
            } else {
                QTextStream stream(&file);
                if (!m_importer->getCodec().isEmpty()) stream.setCodec(m_importer->getCodec().toAscii().constData());

                //Ignore useless lines
                int headerIndex = getCSVHeaderIndex();
                if (headerIndex == -1) {
                    err.setReturnCode(ERR_FAIL);
                    err.setMessage(i18nc("Error message",  "Header not found in CSV file"));
                }

                for (int i = 1; i <= headerIndex; ++i)
                    stream.readLine();

                QList<int> amountIndexes;
                int nb = getCSVMapping().count();
                for (int i = 0; i < nb; ++i) {
                    if (getCSVMapping()[i] == "amount") amountIndexes.push_back(i);
                }
                int nbAmount = amountIndexes.count();

                //Get data column
                QStringList dates;
                QStringList lines;
                bool modeAutoCreditDebit = (!getCSVMapping().contains("sign") && nbAmount == 2);
                if (!err) {
                    int posdate = getCSVMapping().indexOf("date");
                    if (posdate != -1) {
                        QString currentLine;
                        while (!stream.atEnd()) {
                            //Read line
                            QString line = stream.readLine().trimmed();
                            if (!line.isEmpty()) {
                                currentLine += line;

                                //Get date
                                QStringList field = SKGServices::splitCSVLine(currentLine, getCSVSeparator());
                                if (field.count() == 0) {
                                    //This is a multi line csv filename
                                } else {
                                    lines.push_back(currentLine);
                                    currentLine.clear();

                                    if (posdate < field.count()) dates.push_back(field.at(posdate).trimmed());

                                    //Check if all amounts are positive
                                    if (modeAutoCreditDebit) {
                                        for (int i = 0; modeAutoCreditDebit && i < nbAmount; ++i) {
                                            QString s = field.at(amountIndexes.at(i)).trimmed();
                                            if (!s.isEmpty() && SKGServices::stringToDouble(s) < 0) modeAutoCreditDebit = false;
                                        }
                                    }
                                }
                            }
                        }
                    }
                }

                //close file
                file.close();

                //Select dateformat
                QString dateFormat = SKGServices::getDateFormat(dates);
                if (!err && dateFormat.isEmpty()) {
                    err.setReturnCode(ERR_FAIL);
                    err.setMessage(i18nc("Error message",  "Date format not supported"));
                }
                if (!err)
                    err = m_importer->getDocument()->sendMessage(i18nc("An information message",  "Import of '%1' with code '%2' and date format '%3'", iFileName, m_importer->getCodec(), dateFormat));

                //Step 2 done
                if (!err) err = m_importer->getDocument()->stepForward(2);

                //Treat all lines
                if (!err) {
                    int nb = lines.size();
                    err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import operations"), nb);

                    //Save last mapping used in a settings
                    QString mappingDesc;
                    int nbMap = getCSVMapping().count();
                    for (int i = 0; i < nbMap; ++i) {
                        if (i) mappingDesc += '|';
                        mappingDesc += getCSVMapping().at(i);
                    }
                    if (!err) err = m_importer->getDocument()->setParameter("SKG_LAST_CSV_MAPPING_USED", mappingDesc);

                    SKGUnitObject defUnit;
                    SKGAccountObject defAccount;
                    QMap<int, SKGOperationObject> mapGroup;
                    QMap<int, SKGOperationObject> mapOperation;
                    for (int i = 0; !err && i < nb; ++i) {
                        QString currentCategory;
                        SKGOperationObject currentOperation(m_importer->getDocument());
                        SKGSubOperationObject currentSubOperation(m_importer->getDocument());

                        //Valuate mandatory attribute with default value
                        if (getCSVMapping().indexOf("unit") == -1) {
                            err = m_importer->getDefaultUnit(defUnit);
                            if (!err) err = currentOperation.setUnit(defUnit);
                        }
                        if (!err && getCSVMapping().indexOf("account") == -1) {
                            err = m_importer->getDefaultAccount(defAccount);
                            if (!err) err = currentOperation.setParentAccount(defAccount);
                        }

                        QString line = lines.at(i);
                        QByteArray hash = QCryptographicHash::hash(line.toUtf8(), QCryptographicHash::Md5);

                        QString skg_op_original_amount;
                        QStringList atts = SKGServices::splitCSVLine(line, getCSVSeparator());
                        int nbcol = getCSVMapping().count();
                        if (atts.count() < nbcol) {
                            err = SKGError(ERR_INVALIDARG, i18nc("Error message", "Invalid number of columns in line %1. Expected %2. Found %3.",
                                                                 headerIndex + i + 1, nbcol, atts.count()));
                        }
                        int initialBalance = false;
                        int idgroup = 0;
                        int idtransaction = 0;
                        QStringList propertiesAtt;
                        QStringList propertiesVal;
                        double sign = (modeAutoCreditDebit ? -1.0 : 1.0);
                        bool amountSet = false;
                        for (int c = 0; !err && c < nbcol; ++c) {
                            QString col = getCSVMapping()[c];
                            if (!col.isEmpty()) {
                                QString val;
                                if (c >= 0 && c < atts.count()) val = atts.at(c).trimmed();
                                if (col == "date") {
                                    err = currentOperation.setDate(SKGServices::stringToTime(SKGServices::dateToSqlString(val, dateFormat)).date());
                                    if (val == "0000-00-00") initialBalance = true;
                                } else if (col == "number") {
                                    if (!val.isEmpty()) err = currentOperation.setNumber(SKGServices::stringToInt(val));
                                } else if (col == "mode") {
                                    err = currentOperation.setMode(val);
                                } else if (col == "payee") {
                                    SKGPayeeObject payeeObj;
                                    err = SKGPayeeObject::createPayee(m_importer->getDocument(), val, payeeObj);
                                    if (!err) err = currentOperation.setPayee(payeeObj);
                                } else if (col == "comment") {
                                    QString comment = currentOperation.getComment();
                                    if (!comment.isEmpty()) comment += ' ';
                                    comment += val;
                                    err = currentOperation.setComment(comment);
                                    if (!err) err = currentSubOperation.setComment(comment);
                                } else if (col == "status") {
                                    err = currentOperation.setStatus(val == "C" || val == "Y" ?  SKGOperationObject::CHECKED : val == "P" ?  SKGOperationObject::POINTED : SKGOperationObject::NONE);
                                } else if (col == "bookmarked") {
                                    err = currentOperation.bookmark(val == "Y");
                                } else if (col == "idgroup") {
                                    idgroup = SKGServices::stringToInt(val);
                                } else if (col == "idtransaction") {
                                    idtransaction = SKGServices::stringToInt(val);
                                } else if (col == "amount") {
                                    if (!val.isEmpty() && !amountSet) {
                                        amountSet = true;
                                        if (getCSVMapping().contains("quantity")) {
                                            //209705 vvvv
                                            skg_op_original_amount = val;
                                            //209705 ^^^^
                                        } else {
                                            err = currentSubOperation.setQuantity(sign * SKGServices::stringToDouble(val));
                                        }
                                    } else if (modeAutoCreditDebit) sign = 1.0; //Next one will be considered as a credit
                                } else if (col == "quantity") {
                                    err = currentSubOperation.setQuantity(SKGServices::stringToDouble(val));
                                } else if (col == "sign") {
                                    if (QRegExp(m_parameters["mapping_debit"], Qt::CaseInsensitive).indexIn(val) != -1) {
                                        sign = -1;
                                        double cval = currentSubOperation.getQuantity();
                                        if (cval > 0) err = currentSubOperation.setQuantity(-cval);
                                    }
                                } else if (col == "unit") {
                                    //Looking for unit
                                    SKGUnitObject unit(m_importer->getDocument());
                                    if (val != defUnit.getName()) { //For performance
                                        err = unit.setName(val);
                                        if (!err) err = unit.setSymbol(val);
                                        if (!err && unit.load().isFailed())  err = unit.save(false);    //Save only

                                        //This unit is now the default one, it's better for performance
                                        defUnit = unit;
                                    } else {
                                        unit = defUnit;
                                    }

                                    SKGUnitValueObject unitval;
                                    if (!err) err = unit.addUnitValue(unitval);
                                    if (!err) {
                                        int posAmount = getCSVMapping().indexOf("amount");
                                        int posQuantity = getCSVMapping().indexOf("quantity");
                                        if (posAmount != -1 && posQuantity != -1) {
                                            err = unitval.setQuantity(SKGServices::stringToDouble(atts.at(posAmount)) / SKGServices::stringToDouble(atts.at(posQuantity)));
                                        } else {
                                            err = unitval.setQuantity(1);
                                        }
                                    }
                                    if (!err) err = unitval.setDate(now.date());
                                    if (!err) err = unitval.save();
                                    if (!err) err = currentOperation.setUnit(unit);
                                } else if (col == "account") {
                                    //Looking for account
                                    if (val != defAccount.getName()) { //For performance
                                        SKGAccountObject account(m_importer->getDocument());
                                        account.setName(val);
                                        err = account.load();
                                        if (err) {
                                            //Not found, we have to create one
                                            SKGBankObject bank(m_importer->getDocument());
                                            QString name = i18nc("Noun",  "Bank for import %1", postFix);
                                            err = bank.setName(name);
                                            if (!err && bank.load().isFailed()) {
                                                err = bank.save(false);   //Save only
                                                if (!err) err = m_importer->getDocument()->sendMessage(i18nc("An information message",  "Default bank '%1' created for import", name));
                                            }
                                            if (!err) err = bank.addAccount(account);
                                            if (!err) err = account.setName(val);
                                            if (!err && account.load().isFailed())  err = account.save(false);     //Save only
                                        }

                                        //This account is now the default one, it's better for performance
                                        defAccount = account;
                                    }
                                    if (!err) err = currentOperation.setParentAccount(defAccount);
                                } else if (col == "category") {
                                    //Set Category
                                    if (!val.isEmpty()) {
                                        //Prepare val
                                        val.replace('/', OBJECTSEPARATOR);
                                        val.replace(':', OBJECTSEPARATOR);
                                        val.replace(',', OBJECTSEPARATOR);
                                        val.replace(';', OBJECTSEPARATOR);
                                        //Get previous category
                                        if (!currentCategory.isEmpty()) val = currentCategory % OBJECTSEPARATOR % val;
                                        currentCategory = val;

                                        //Create and set category
                                        SKGCategoryObject Category;
                                        err = SKGCategoryObject::createPathCategory(m_importer->getDocument(), val, Category);
                                        if (!err)  err = currentSubOperation.setCategory(Category);
                                    }
                                } else {
                                    //A property
                                    propertiesAtt.push_back(col);
                                    propertiesVal.push_back(val);
                                }
                            }
                        }

                        if (!err && initialBalance) {
                            //Specific values for initial balance
                            err = currentOperation.setStatus(SKGOperationObject::CHECKED);
                            if (!err) err = currentOperation.setAttribute("d_date", "0000-00-00");
                        }
                        if (!err) err = currentOperation.setImportID(hash.toHex());
                        if (!err) {
                            if (idtransaction != 0) {
                                if (mapOperation.contains(idtransaction)) {
                                    currentOperation = mapOperation[idtransaction];
                                    skg_op_original_amount = "";
                                } else {
                                    err = currentOperation.save();
                                    mapOperation[idtransaction] = currentOperation;
                                }
                            } else {
                                err = currentOperation.save(false);   //Save only
                            }
                        }
                        if (!err && idgroup != 0) {
                            if (mapGroup.contains(idgroup)) {
                                err = currentOperation.setGroupOperation(mapGroup[idgroup]);
                                if (!err) err = currentOperation.save();
                            }
                            mapGroup[idgroup] = currentOperation;
                        }

                        if (!err) err = currentSubOperation.setParentOperation(currentOperation);
                        if (!err) err = currentSubOperation.save(false, false);     //Save only without reload

                        //209705 vvvv
                        if (!err && !skg_op_original_amount.isEmpty()) {
                            err = currentOperation.setProperty("SKG_OP_ORIGINAL_AMOUNT", skg_op_original_amount);
                        }
                        //209705 ^^^^

                        //Add properties
                        int nbp = propertiesAtt.count();
                        for (int p = 0; !err && p < nbp; ++p) {
                            err = currentOperation.setProperty(propertiesAtt.at(p) , propertiesVal.at(p));
                        }

                        if (!err && i % 20 == 0) err = m_importer->getDocument()->executeSqliteOrder("ANALYZE");
                        if (!err) err = m_importer->getDocument()->stepForward(i + 1);
                    }

                    if (!err) err = m_importer->getDocument()->endTransaction(true);
                    else  m_importer->getDocument()->endTransaction(false);

                    //Lines treated
                    if (!err) err = m_importer->getDocument()->stepForward(3);
                }
            }
        }
    }
    if (!err) err = m_importer->getDocument()->endTransaction(true);
    else  m_importer->getDocument()->endTransaction(false);

    return err;
}


bool SKGImportPluginCsv::isExportPossible()
{
    SKGTRACEIN(10, "SKGImportPluginCsv::isExportPossible");
    return (!m_importer ? true : QFileInfo(m_importer->getFileName()).suffix().toUpper() == "CSV");
}

SKGError SKGImportPluginCsv::exportFile()
{
    if (!m_importer) return SKGError(ERR_ABORT, i18nc("Error message", "Invalid parameters"));
    SKGError err;
    SKGTRACEINRC(2, "SKGImportCsv::exportFile", err);
    QString iFileName = m_importer->getFileName();
    SKGTRACEL(10) << "Input filename=" << iFileName << endl;

    //Open file
    KSaveFile file(iFileName);
    if (!file.open()) {
        err.setReturnCode(ERR_INVALIDARG);
        err.setMessage(i18nc("Error message",  "Save file '%1' failed", iFileName));
    } else {
        QTextStream out(&file);
        if (!m_importer->getCodec().isEmpty()) out.setCodec(m_importer->getCodec().toAscii().constData());
        err = m_importer->getDocument()->dumpSelectSqliteOrder(
                  "SELECT v.d_date as date, v.t_ACCOUNT as account, v.i_number as number, v.t_mode as mode, "
                  "v.t_PAYEE as payee, v.t_REALCOMMENT as comment, ROUND(v.f_REALQUANTITY, u.i_nbdecimal) as quantity, "
                  "v.t_UNIT as unit, ROUND(v.f_REALCURRENTAMOUNT, u.i_nbdecimal) as amount, v.t_TYPEEXPENSE as sign, v.t_REALCATEGORY as category, v.t_status as status, "
                  "v.t_bookmarked as bookmarked, v.i_SUBOPID id, v.id idtransaction, v.i_group_id idgroup "
                  "FROM v_operation_consolidated as v, unit as u WHERE v.rc_unit_id=u.id ORDER BY v.d_date, v.id, v.i_SUBOPID", &out, SKGServices::DUMP_CSV);
    }

    //Close file
    file.finalize();
    file.close();

    return err;
}

QString SKGImportPluginCsv::getMimeTypeFilter() const
{
    return "*.csv|" % i18nc("A file format", "CSV file");
}

SKGError SKGImportPluginCsv::importCSVUnit()
{
    SKGError err;
    SKGTRACEINRC(2, "SKGImportPluginCsv::importCSVUnit", err);
    SKGTRACEL(10) << "Input filename=" << m_importer->getFileName() << endl;

    if (m_importer->getDocument()) {
        //Begin transaction
        err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import units"), 3);
        if (!err) {
            //File name is the name of the unit
            QFileInfo fInfo(m_importer->getFileName());
            QString unitName = fInfo.baseName();

            //Default mapping
            if (getCSVMapping().count() == 0) {
                err = setCSVMapping(NULL);
                if (!err) err = m_importer->getDocument()->sendMessage(i18nc("An information message",  "Use automatic search of the columns"));
            }

            //Step 1 done
            if (!err) err = m_importer->getDocument()->stepForward(1);

            //Open file
            if (!err) {
                QFile file(m_importer->getFileName());
                if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
                    err.setReturnCode(ERR_INVALIDARG);
                    err.setMessage(i18nc("Error message",  "Open file '%1' failed", m_importer->getFileName()));
                } else {
                    QTextStream stream(&file);
                    if (!m_importer->getCodec().isEmpty()) stream.setCodec(m_importer->getCodec().toAscii().constData());

                    //Ignore useless lines
                    int headerIndex = getCSVHeaderIndex();
                    for (int i = 1; i <= headerIndex; ++i)
                        stream.readLine();

                    //Get data column
                    QStringList dates;
                    QStringList lines;
                    int posdate = m_csvMapping.indexOf("date");
                    if (posdate != -1) {
                        while (!stream.atEnd()) {
                            //Read line
                            QString line = stream.readLine().trimmed();
                            if (!line.isEmpty()) {
                                lines.push_back(line);

                                //Get date
                                QStringList field = SKGServices::splitCSVLine(line);
                                if (posdate < field.count()) dates.push_back(field.at(posdate));
                            }
                        }
                    }

                    //close file
                    file.close();

                    //Select dateformat
                    QString dateFormat = SKGServices::getDateFormat(dates);
                    if (dateFormat.isEmpty()) {
                        err.setReturnCode(ERR_FAIL);
                        err.setMessage(i18nc("Error message",  "Date format not supported"));
                    }
                    if (!err)
                        err = m_importer->getDocument()->sendMessage(i18nc("An information message",  "Import of '%1' with code '%2' and date format '%3'", m_importer->getFileName(), m_importer->getCodec(), dateFormat));

                    //Step 2 done
                    if (!err) err = m_importer->getDocument()->stepForward(2);

                    //Treat all lines
                    if (!err) {
                        int nb = lines.size();
                        err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import units"), nb);

                        //Save last mapping used in a settings
                        QString mappingDesc;
                        int nbMap = m_csvMapping.count();
                        for (int i = 0; i < nbMap; ++i) {
                            if (i) mappingDesc += '|';
                            mappingDesc += m_csvMapping.at(i);
                        }
                        if (!err) err = m_importer->getDocument()->setParameter("SKG_LAST_CSV_UNIT_MAPPING_USED", mappingDesc);

                        int posdate = m_csvMapping.indexOf("date");
                        int posvalue = m_csvMapping.indexOf("amount");
                        if (posdate != -1 && posvalue != -1) {
                            for (int i = 0; !err && i < nb; ++i) {
                                QStringList atts = SKGServices::splitCSVLine(lines.at(i));
                                err = m_importer->getDocument()->addOrModifyUnitValue(unitName,
                                        SKGServices::stringToTime(SKGServices::dateToSqlString(atts.at(posdate), dateFormat)).date(),
                                        SKGServices::stringToDouble(atts.at(posvalue)));

                                if (!err) err = m_importer->getDocument()->stepForward(i + 1);
                            }
                        }

                        if (!err) err = m_importer->getDocument()->endTransaction(true);
                        else  m_importer->getDocument()->endTransaction(false);

                        //Lines treated
                        if (!err) err = m_importer->getDocument()->stepForward(3);
                    }
                }
            }
        }
        if (!err) err = m_importer->getDocument()->endTransaction(true);
        else  m_importer->getDocument()->endTransaction(false);
    }
    return err;
}
#include "skgimportplugincsv.moc"
