diff -Nru kio-recoll-1.30.1/aspell/aspell-local.h kio-recoll-1.31.0/aspell/aspell-local.h --- kio-recoll-1.30.1/aspell/aspell-local.h 2020-05-30 13:52:40.000000000 +0000 +++ kio-recoll-1.31.0/aspell/aspell-local.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,729 +0,0 @@ -/* Automatically generated file. Do not edit directly. */ - -/* This file is part of The New Aspell - * Copyright (C) 2001-2002 by Kevin Atkinson under the GNU LGPL - * license version 2.0 or 2.1. You should have received a copy of the - * LGPL license along with this library if you did not you can find it - * at http://www.gnu.org/. */ - -#ifndef ASPELL_ASPELL__H -#define ASPELL_ASPELL__H - -#ifdef __cplusplus -extern "C" { -#endif - -/******************************* type id *******************************/ - - -union AspellTypeId { - - unsigned int num; - - char str[4]; - -}; - - -typedef union AspellTypeId AspellTypeId; - - -/************************** mutable container **************************/ - - -typedef struct AspellMutableContainer AspellMutableContainer; - - -int aspell_mutable_container_add(struct AspellMutableContainer * ths, const char * to_add); - -int aspell_mutable_container_remove(struct AspellMutableContainer * ths, const char * to_rem); - -void aspell_mutable_container_clear(struct AspellMutableContainer * ths); - -struct AspellMutableContainer * aspell_mutable_container_to_mutable_container(struct AspellMutableContainer * ths); - - - -/******************************* key info *******************************/ - - - -enum AspellKeyInfoType {AspellKeyInfoString, AspellKeyInfoInt, AspellKeyInfoBool, AspellKeyInfoList}; -typedef enum AspellKeyInfoType AspellKeyInfoType; - - -struct AspellKeyInfo { - - /* The name of the key. */ - const char * name; - - /* The key type. */ - enum AspellKeyInfoType type; - - /* The default value of the key. */ - const char * def; - - /* A brief description of the key or NULL if internal value. */ - const char * desc; - - int flags; - - int other_data; - -}; - - -typedef struct AspellKeyInfo AspellKeyInfo; - - -/******************************** config ********************************/ - - -typedef struct AspellKeyInfoEnumeration AspellKeyInfoEnumeration; - - -int aspell_key_info_enumeration_at_end(const struct AspellKeyInfoEnumeration * ths); - -const struct AspellKeyInfo * aspell_key_info_enumeration_next(struct AspellKeyInfoEnumeration * ths); - -void delete_aspell_key_info_enumeration(struct AspellKeyInfoEnumeration * ths); - -struct AspellKeyInfoEnumeration * aspell_key_info_enumeration_clone(const struct AspellKeyInfoEnumeration * ths); - -void aspell_key_info_enumeration_assign(struct AspellKeyInfoEnumeration * ths, const struct AspellKeyInfoEnumeration * other); - - - -typedef struct AspellConfig AspellConfig; - - -struct AspellConfig * new_aspell_config(); - -void delete_aspell_config(struct AspellConfig * ths); - -struct AspellConfig * aspell_config_clone(const struct AspellConfig * ths); - -void aspell_config_assign(struct AspellConfig * ths, const struct AspellConfig * other); - -unsigned int aspell_config_error_number(const struct AspellConfig * ths); - -const char * aspell_config_error_message(const struct AspellConfig * ths); - -const struct AspellError * aspell_config_error(const struct AspellConfig * ths); - -/* Sets extra keys which this config class should - * accept. begin and end are expected to point to - * the beginning and ending of an array of Aspell - * Key Info. */ -void aspell_config_set_extra(struct AspellConfig * ths, const struct AspellKeyInfo * begin, const struct AspellKeyInfo * end); - -/* Returns the KeyInfo object for the - * corresponding key or returns NULL and sets - * error_num to PERROR_UNKNOWN_KEY if the key is - * not valid. The pointer returned is valid for - * the lifetime of the object. */ -const struct AspellKeyInfo * aspell_config_keyinfo(struct AspellConfig * ths, const char * key); - -/* Returns a newly allocated enumeration of all - * the possible objects this config class uses. */ -struct AspellKeyInfoEnumeration * aspell_config_possible_elements(struct AspellConfig * ths, int include_extra); - -/* Returns the default value for given key which - * may involve substituting variables, thus it is - * not the same as keyinfo(key)->def returns NULL - * and sets error_num to PERROR_UNKNOWN_KEY if - * the key is not valid. Uses the temporary - * string. */ -const char * aspell_config_get_default(struct AspellConfig * ths, const char * key); - -/* Returns a newly allocated enumeration of all - * the key/value pairs. This DOES not include ones - * which are set to their default values. */ -struct AspellStringPairEnumeration * aspell_config_elements(struct AspellConfig * ths); - -/* Inserts an item, if the item already exists it - * will be replaced. Returns TRUE if it succeeded - * or FALSE on error. If the key in not valid it - * sets error_num to PERROR_UNKNOWN_KEY, if the - * value is not valid it will set error_num to - * PERROR_BAD_VALUE, if the value can not be - * changed it sets error_num to - * PERROR_CANT_CHANGE_VALUE, and if the value is - * a list and you are trying to set its directory, - * it sets error_num to PERROR_LIST_SET */ -int aspell_config_replace(struct AspellConfig * ths, const char * key, const char * value); - -/* Remove a key and returns TRUE if it exists - * otherwise return FALSE. This effectively sets - * the key to its default value. Calling replace - * with a value of "" will also call - * remove. If the key does not exist then it sets - * error_num to 0 or PERROR_NOT, if the key is - * not valid then it sets error_num to - * PERROR_UNKNOWN_KEY, if the value can not be - * changed then it sets error_num to - * PERROR_CANT_CHANGE_VALUE */ -int aspell_config_remove(struct AspellConfig * ths, const char * key); - -int aspell_config_have(const struct AspellConfig * ths, const char * key); - -/* Returns NULL on error. */ -const char * aspell_config_retrieve(struct AspellConfig * ths, const char * key); - -int aspell_config_retrieve_list(struct AspellConfig * ths, const char * key, struct AspellMutableContainer * lst); - -/* Return -1 on error, 0 if false, 1 if true. */ -int aspell_config_retrieve_bool(struct AspellConfig * ths, const char * key); - -/* Return -1 on error. */ -int aspell_config_retrieve_int(struct AspellConfig * ths, const char * key); - - - -/******************************** error ********************************/ - - -struct AspellError { - - const char * mesg; - - const struct AspellErrorInfo * err; - -}; - - -typedef struct AspellError AspellError; - -int aspell_error_is_a(const struct AspellError * ths, const struct AspellErrorInfo * e); - - -struct AspellErrorInfo { - - const struct AspellErrorInfo * isa; - - const char * mesg; - - unsigned int num_parms; - - const char * parms[3]; - -}; - - -typedef struct AspellErrorInfo AspellErrorInfo; - - -/**************************** can have error ****************************/ - - -typedef struct AspellCanHaveError AspellCanHaveError; - - -unsigned int aspell_error_number(const struct AspellCanHaveError * ths); - -const char * aspell_error_message(const struct AspellCanHaveError * ths); - -const struct AspellError * aspell_error(const struct AspellCanHaveError * ths); - -void delete_aspell_can_have_error(struct AspellCanHaveError * ths); - - - -/******************************** errors ********************************/ - - -extern const struct AspellErrorInfo * const aerror_other; -extern const struct AspellErrorInfo * const aerror_operation_not_supported; -extern const struct AspellErrorInfo * const aerror_cant_copy; -extern const struct AspellErrorInfo * const aerror_unimplemented_method; -extern const struct AspellErrorInfo * const aerror_file; -extern const struct AspellErrorInfo * const aerror_cant_open_file; -extern const struct AspellErrorInfo * const aerror_cant_read_file; -extern const struct AspellErrorInfo * const aerror_cant_write_file; -extern const struct AspellErrorInfo * const aerror_invalid_name; -extern const struct AspellErrorInfo * const aerror_bad_file_format; -extern const struct AspellErrorInfo * const aerror_dir; -extern const struct AspellErrorInfo * const aerror_cant_read_dir; -extern const struct AspellErrorInfo * const aerror_config; -extern const struct AspellErrorInfo * const aerror_unknown_key; -extern const struct AspellErrorInfo * const aerror_cant_change_value; -extern const struct AspellErrorInfo * const aerror_bad_key; -extern const struct AspellErrorInfo * const aerror_bad_value; -extern const struct AspellErrorInfo * const aerror_duplicate; -extern const struct AspellErrorInfo * const aerror_key_not_string; -extern const struct AspellErrorInfo * const aerror_key_not_int; -extern const struct AspellErrorInfo * const aerror_key_not_bool; -extern const struct AspellErrorInfo * const aerror_key_not_list; -extern const struct AspellErrorInfo * const aerror_no_value_reset; -extern const struct AspellErrorInfo * const aerror_no_value_enable; -extern const struct AspellErrorInfo * const aerror_no_value_disable; -extern const struct AspellErrorInfo * const aerror_no_value_clear; -extern const struct AspellErrorInfo * const aerror_language_related; -extern const struct AspellErrorInfo * const aerror_unknown_language; -extern const struct AspellErrorInfo * const aerror_unknown_soundslike; -extern const struct AspellErrorInfo * const aerror_language_not_supported; -extern const struct AspellErrorInfo * const aerror_no_wordlist_for_lang; -extern const struct AspellErrorInfo * const aerror_mismatched_language; -extern const struct AspellErrorInfo * const aerror_affix; -extern const struct AspellErrorInfo * const aerror_corrupt_affix; -extern const struct AspellErrorInfo * const aerror_invalid_cond; -extern const struct AspellErrorInfo * const aerror_invalid_cond_strip; -extern const struct AspellErrorInfo * const aerror_incorrect_encoding; -extern const struct AspellErrorInfo * const aerror_encoding; -extern const struct AspellErrorInfo * const aerror_unknown_encoding; -extern const struct AspellErrorInfo * const aerror_encoding_not_supported; -extern const struct AspellErrorInfo * const aerror_conversion_not_supported; -extern const struct AspellErrorInfo * const aerror_pipe; -extern const struct AspellErrorInfo * const aerror_cant_create_pipe; -extern const struct AspellErrorInfo * const aerror_process_died; -extern const struct AspellErrorInfo * const aerror_bad_input; -extern const struct AspellErrorInfo * const aerror_invalid_string; -extern const struct AspellErrorInfo * const aerror_invalid_word; -extern const struct AspellErrorInfo * const aerror_invalid_affix; -extern const struct AspellErrorInfo * const aerror_inapplicable_affix; -extern const struct AspellErrorInfo * const aerror_unknown_unichar; -extern const struct AspellErrorInfo * const aerror_word_list_flags; -extern const struct AspellErrorInfo * const aerror_invalid_flag; -extern const struct AspellErrorInfo * const aerror_conflicting_flags; -extern const struct AspellErrorInfo * const aerror_version_control; -extern const struct AspellErrorInfo * const aerror_bad_version_string; -extern const struct AspellErrorInfo * const aerror_filter; -extern const struct AspellErrorInfo * const aerror_cant_dlopen_file; -extern const struct AspellErrorInfo * const aerror_empty_filter; -extern const struct AspellErrorInfo * const aerror_no_such_filter; -extern const struct AspellErrorInfo * const aerror_confusing_version; -extern const struct AspellErrorInfo * const aerror_bad_version; -extern const struct AspellErrorInfo * const aerror_identical_option; -extern const struct AspellErrorInfo * const aerror_options_only; -extern const struct AspellErrorInfo * const aerror_invalid_option_modifier; -extern const struct AspellErrorInfo * const aerror_cant_describe_filter; -extern const struct AspellErrorInfo * const aerror_filter_mode_file; -extern const struct AspellErrorInfo * const aerror_mode_option_name; -extern const struct AspellErrorInfo * const aerror_no_filter_to_option; -extern const struct AspellErrorInfo * const aerror_bad_mode_key; -extern const struct AspellErrorInfo * const aerror_expect_mode_key; -extern const struct AspellErrorInfo * const aerror_mode_version_requirement; -extern const struct AspellErrorInfo * const aerror_confusing_mode_version; -extern const struct AspellErrorInfo * const aerror_bad_mode_version; -extern const struct AspellErrorInfo * const aerror_missing_magic_expression; -extern const struct AspellErrorInfo * const aerror_empty_file_ext; -extern const struct AspellErrorInfo * const aerror_filter_mode_expand; -extern const struct AspellErrorInfo * const aerror_unknown_mode; -extern const struct AspellErrorInfo * const aerror_mode_extend_expand; -extern const struct AspellErrorInfo * const aerror_filter_mode_magic; -extern const struct AspellErrorInfo * const aerror_file_magic_pos; -extern const struct AspellErrorInfo * const aerror_file_magic_range; -extern const struct AspellErrorInfo * const aerror_missing_magic; -extern const struct AspellErrorInfo * const aerror_bad_magic; -extern const struct AspellErrorInfo * const aerror_expression; -extern const struct AspellErrorInfo * const aerror_invalid_expression; - - -/******************************* speller *******************************/ - - -typedef struct AspellSpeller AspellSpeller; - - -struct AspellCanHaveError * new_aspell_speller(struct AspellConfig * config); - -struct AspellSpeller * to_aspell_speller(struct AspellCanHaveError * obj); - -void delete_aspell_speller(struct AspellSpeller * ths); - -unsigned int aspell_speller_error_number(const struct AspellSpeller * ths); - -const char * aspell_speller_error_message(const struct AspellSpeller * ths); - -const struct AspellError * aspell_speller_error(const struct AspellSpeller * ths); - -struct AspellConfig * aspell_speller_config(struct AspellSpeller * ths); - -/* Returns 0 if it is not in the dictionary, - * 1 if it is, or -1 on error. */ -int aspell_speller_check(struct AspellSpeller * ths, const char * word, int word_size); - -/* Add this word to your own personal word list. */ -int aspell_speller_add_to_personal(struct AspellSpeller * ths, const char * word, int word_size); - -/* Add this word to the current spelling session. */ -int aspell_speller_add_to_session(struct AspellSpeller * ths, const char * word, int word_size); - -/* This is your own personal word list file plus - * any extra words added during this session to - * your own personal word list. */ -const struct AspellWordList * aspell_speller_personal_word_list(struct AspellSpeller * ths); - -/* This is a list of words added to this session - * that are not in the main word list or in your - * own personal list but are considered valid for - * this spelling session. */ -const struct AspellWordList * aspell_speller_session_word_list(struct AspellSpeller * ths); - -/* This is the main list of words used during this - * spelling session. */ -const struct AspellWordList * aspell_speller_main_word_list(struct AspellSpeller * ths); - -int aspell_speller_save_all_word_lists(struct AspellSpeller * ths); - -int aspell_speller_clear_session(struct AspellSpeller * ths); - -/* Return NULL on error. - * The word list returned by suggest is only - * valid until the next call to suggest. */ -const struct AspellWordList * aspell_speller_suggest(struct AspellSpeller * ths, const char * word, int word_size); - -int aspell_speller_store_replacement(struct AspellSpeller * ths, const char * mis, int mis_size, const char * cor, int cor_size); - - - -/******************************** filter ********************************/ - - -typedef struct AspellFilter AspellFilter; - - -void delete_aspell_filter(struct AspellFilter * ths); - -unsigned int aspell_filter_error_number(const struct AspellFilter * ths); - -const char * aspell_filter_error_message(const struct AspellFilter * ths); - -const struct AspellError * aspell_filter_error(const struct AspellFilter * ths); - -struct AspellFilter * to_aspell_filter(struct AspellCanHaveError * obj); - - - -/*************************** document checker ***************************/ - - -struct AspellToken { - - unsigned int offset; - - unsigned int len; - -}; - - -typedef struct AspellToken AspellToken; - - -typedef struct AspellDocumentChecker AspellDocumentChecker; - - -void delete_aspell_document_checker(struct AspellDocumentChecker * ths); - -unsigned int aspell_document_checker_error_number(const struct AspellDocumentChecker * ths); - -const char * aspell_document_checker_error_message(const struct AspellDocumentChecker * ths); - -const struct AspellError * aspell_document_checker_error(const struct AspellDocumentChecker * ths); - -/* Creates a new document checker. - * The speller class is expected to last until - * this class is destroyed. - * If config is given it will be used to override - * any relevent options set by this speller class. - * The config class is not once this function is done. - * If filter is given then it will take ownership of - * the filter class and use it to do the filtering. - * You are expected to free the checker when done. */ -struct AspellCanHaveError * new_aspell_document_checker(struct AspellSpeller * speller); - -struct AspellDocumentChecker * to_aspell_document_checker(struct AspellCanHaveError * obj); - -/* Reset the internal state of the filter. - * Should be called whenever a new document is - * being filtered. */ -void aspell_document_checker_reset(struct AspellDocumentChecker * ths); - -/* Process a string. - * The string passed in should only be split on - * white space characters. Furthermore, between - * calls to reset, each string should be passed - * in exactly once and in the order they appeared - * in the document. Passing in strings out of - * order, skipping strings or passing them in - * more than once may lead to undefined results. */ -void aspell_document_checker_process(struct AspellDocumentChecker * ths, const char * str, int size); - -/* Returns the next misspelled word in the - * processed string. If there are no more - * misspelled words, then token.word will be - * NULL and token.size will be 0 */ -struct AspellToken aspell_document_checker_next_misspelling(struct AspellDocumentChecker * ths); - -/* Returns the underlying filter class. */ -struct AspellFilter * aspell_document_checker_filter(struct AspellDocumentChecker * ths); - - - -/****************************** word list ******************************/ - - -typedef struct AspellWordList AspellWordList; - - -int aspell_word_list_empty(const struct AspellWordList * ths); - -unsigned int aspell_word_list_size(const struct AspellWordList * ths); - -struct AspellStringEnumeration * aspell_word_list_elements(const struct AspellWordList * ths); - - - -/************************** string enumeration **************************/ - - -typedef struct AspellStringEnumeration AspellStringEnumeration; - - -void delete_aspell_string_enumeration(struct AspellStringEnumeration * ths); - -struct AspellStringEnumeration * aspell_string_enumeration_clone(const struct AspellStringEnumeration * ths); - -void aspell_string_enumeration_assign(struct AspellStringEnumeration * ths, const struct AspellStringEnumeration * other); - -int aspell_string_enumeration_at_end(const struct AspellStringEnumeration * ths); - -const char * aspell_string_enumeration_next(struct AspellStringEnumeration * ths); - - - -/********************************* info *********************************/ - - -struct AspellModuleInfo { - - const char * name; - - double order_num; - - const char * lib_dir; - - struct AspellStringList * dict_dirs; - - struct AspellStringList * dict_exts; - -}; - - -typedef struct AspellModuleInfo AspellModuleInfo; - - -struct AspellDictInfo { - - /* The Name to identify this dictionary by. */ - const char * name; - - /* The language code to identify this dictionary. - * A two letter UPPER-CASE ISO 639 language code - * and an optional two letter ISO 3166 country - * code after a dash or underscore. */ - const char * code; - - /* Any extra information to distinguish this - * variety of dictionary from other dictionaries - * which may have the same language and size. */ - const char * jargon; - - int size; - - /* A two char digit code describing the size of - * the dictionary: 10=tiny, 20=really small, - * 30=small, 40=med-small, 50=med, 60=med-large, - * 70=large, 80=huge, 90=insane. Please check - * the README in aspell-lang-200?????.tar.bz2 or - * see SCOWL (http://wordlist.sourceforge.net) - * for an example of how these sizes are used. */ - const char * size_str; - - struct AspellModuleInfo * module; - -}; - - -typedef struct AspellDictInfo AspellDictInfo; - - -typedef struct AspellModuleInfoList AspellModuleInfoList; - - -struct AspellModuleInfoList * get_aspell_module_info_list(struct AspellConfig * config); - -int aspell_module_info_list_empty(const struct AspellModuleInfoList * ths); - -unsigned int aspell_module_info_list_size(const struct AspellModuleInfoList * ths); - -struct AspellModuleInfoEnumeration * aspell_module_info_list_elements(const struct AspellModuleInfoList * ths); - - - -typedef struct AspellDictInfoList AspellDictInfoList; - - -struct AspellDictInfoList * get_aspell_dict_info_list(struct AspellConfig * config); - -int aspell_dict_info_list_empty(const struct AspellDictInfoList * ths); - -unsigned int aspell_dict_info_list_size(const struct AspellDictInfoList * ths); - -struct AspellDictInfoEnumeration * aspell_dict_info_list_elements(const struct AspellDictInfoList * ths); - - - -typedef struct AspellModuleInfoEnumeration AspellModuleInfoEnumeration; - - -int aspell_module_info_enumeration_at_end(const struct AspellModuleInfoEnumeration * ths); - -const struct AspellModuleInfo * aspell_module_info_enumeration_next(struct AspellModuleInfoEnumeration * ths); - -void delete_aspell_module_info_enumeration(struct AspellModuleInfoEnumeration * ths); - -struct AspellModuleInfoEnumeration * aspell_module_info_enumeration_clone(const struct AspellModuleInfoEnumeration * ths); - -void aspell_module_info_enumeration_assign(struct AspellModuleInfoEnumeration * ths, const struct AspellModuleInfoEnumeration * other); - - - -typedef struct AspellDictInfoEnumeration AspellDictInfoEnumeration; - - -int aspell_dict_info_enumeration_at_end(const struct AspellDictInfoEnumeration * ths); - -const struct AspellDictInfo * aspell_dict_info_enumeration_next(struct AspellDictInfoEnumeration * ths); - -void delete_aspell_dict_info_enumeration(struct AspellDictInfoEnumeration * ths); - -struct AspellDictInfoEnumeration * aspell_dict_info_enumeration_clone(const struct AspellDictInfoEnumeration * ths); - -void aspell_dict_info_enumeration_assign(struct AspellDictInfoEnumeration * ths, const struct AspellDictInfoEnumeration * other); - - - -/***************************** string list *****************************/ - - -typedef struct AspellStringList AspellStringList; - - -struct AspellStringList * new_aspell_string_list(); - -int aspell_string_list_empty(const struct AspellStringList * ths); - -unsigned int aspell_string_list_size(const struct AspellStringList * ths); - -struct AspellStringEnumeration * aspell_string_list_elements(const struct AspellStringList * ths); - -int aspell_string_list_add(struct AspellStringList * ths, const char * to_add); - -int aspell_string_list_remove(struct AspellStringList * ths, const char * to_rem); - -void aspell_string_list_clear(struct AspellStringList * ths); - -struct AspellMutableContainer * aspell_string_list_to_mutable_container(struct AspellStringList * ths); - -void delete_aspell_string_list(struct AspellStringList * ths); - -struct AspellStringList * aspell_string_list_clone(const struct AspellStringList * ths); - -void aspell_string_list_assign(struct AspellStringList * ths, const struct AspellStringList * other); - - - -/****************************** string map ******************************/ - - -typedef struct AspellStringMap AspellStringMap; - - -struct AspellStringMap * new_aspell_string_map(); - -int aspell_string_map_add(struct AspellStringMap * ths, const char * to_add); - -int aspell_string_map_remove(struct AspellStringMap * ths, const char * to_rem); - -void aspell_string_map_clear(struct AspellStringMap * ths); - -struct AspellMutableContainer * aspell_string_map_to_mutable_container(struct AspellStringMap * ths); - -void delete_aspell_string_map(struct AspellStringMap * ths); - -struct AspellStringMap * aspell_string_map_clone(const struct AspellStringMap * ths); - -void aspell_string_map_assign(struct AspellStringMap * ths, const struct AspellStringMap * other); - -int aspell_string_map_empty(const struct AspellStringMap * ths); - -unsigned int aspell_string_map_size(const struct AspellStringMap * ths); - -struct AspellStringPairEnumeration * aspell_string_map_elements(const struct AspellStringMap * ths); - -/* Insert a new element. - * Will NOT overwrite an existing entry. - * Returns FALSE if the element already exists. */ -int aspell_string_map_insert(struct AspellStringMap * ths, const char * key, const char * value); - -/* Insert a new element. - * Will overwrite an existing entry. - * Always returns TRUE. */ -int aspell_string_map_replace(struct AspellStringMap * ths, const char * key, const char * value); - -/* Looks up an element and returns the value. - * Returns NULL if the element does not exist. - * Returns an empty string if the element exists - * but has a NULL value. */ -const char * aspell_string_map_lookup(const struct AspellStringMap * ths, const char * key); - - - -/***************************** string pair *****************************/ - - -struct AspellStringPair { - - const char * first; - - const char * second; - -}; - - -typedef struct AspellStringPair AspellStringPair; - - -/*********************** string pair enumeration ***********************/ - - -typedef struct AspellStringPairEnumeration AspellStringPairEnumeration; - - -int aspell_string_pair_enumeration_at_end(const struct AspellStringPairEnumeration * ths); - -struct AspellStringPair aspell_string_pair_enumeration_next(struct AspellStringPairEnumeration * ths); - -void delete_aspell_string_pair_enumeration(struct AspellStringPairEnumeration * ths); - -struct AspellStringPairEnumeration * aspell_string_pair_enumeration_clone(const struct AspellStringPairEnumeration * ths); - -void aspell_string_pair_enumeration_assign(struct AspellStringPairEnumeration * ths, const struct AspellStringPairEnumeration * other); - - - -/******************************** cache ********************************/ - - -/* Reset the global cache(s) so that cache queries will - * create a new object. If existing objects are still in - * use they are not deleted. If which is NULL then all - * caches will be reset. Current caches are "encode", - * "decode", "dictionary", "language", and "keyboard". */ -int aspell_reset_cache(const char * which); - -#ifdef __cplusplus -} -#endif -#endif /* ASPELL_ASPELL__H */ diff -Nru kio-recoll-1.30.1/aspell/rclaspell.cpp kio-recoll-1.31.0/aspell/rclaspell.cpp --- kio-recoll-1.30.1/aspell/rclaspell.cpp 2020-09-05 07:43:16.000000000 +0000 +++ kio-recoll-1.31.0/aspell/rclaspell.cpp 2021-04-23 11:53:12.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2006-2019 J.F.Dockes +/* Copyright (C) 2006-2021 J.F.Dockes * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -18,8 +18,6 @@ #ifdef RCL_USE_ASPELL -#include ASPELL_INCLUDE - #include #include @@ -34,72 +32,11 @@ using namespace std; -// Aspell library entry points -class AspellApi { -public: - struct AspellConfig *(*new_aspell_config)(); - int (*aspell_config_replace)(struct AspellConfig *, const char * key, - const char * value); - struct AspellCanHaveError *(*new_aspell_speller)(struct AspellConfig *); - void (*delete_aspell_config)(struct AspellConfig *); - void (*delete_aspell_can_have_error)(struct AspellCanHaveError *); - struct AspellSpeller * (*to_aspell_speller)(struct AspellCanHaveError *); - struct AspellConfig * (*aspell_speller_config)(struct AspellSpeller *); - const struct AspellWordList * (*aspell_speller_suggest) - (struct AspellSpeller *, const char *, int); - int (*aspell_speller_check)(struct AspellSpeller *, const char *, int); - struct AspellStringEnumeration * (*aspell_word_list_elements) - (const struct AspellWordList * ths); - const char * (*aspell_string_enumeration_next) - (struct AspellStringEnumeration * ths); - void (*delete_aspell_string_enumeration)(struct AspellStringEnumeration *); - const struct AspellError *(*aspell_error) - (const struct AspellCanHaveError *); - const char *(*aspell_error_message)(const struct AspellCanHaveError *); - const char *(*aspell_speller_error_message)(const struct AspellSpeller *); - void (*delete_aspell_speller)(struct AspellSpeller *); - -}; -static AspellApi aapi; -static std::mutex o_aapi_mutex; - -#define NMTOPTR(NM, TP) \ - if ((aapi.NM = TP dlib_sym(m_data->m_handle, #NM)) == 0) { \ - badnames += #NM + string(" "); \ - } - -static const vector aspell_lib_suffixes { -#if defined(__APPLE__) - ".15.dylib", - ".dylib", -#elif defined(_WIN32) - "-15.dll", -#else - ".so", - ".so.15", -#endif -}; - // Private rclaspell data class AspellData { public: - ~AspellData() { - LOGDEB2("~AspellData\n" ); - if (m_handle) { - dlib_close(m_handle); - m_handle = nullptr; - } - if (m_speller) { - // Dumps core if I do this?? - //aapi.delete_aspell_speller(m_speller); - m_speller = nullptr; - LOGDEB2("~AspellData: speller done\n" ); - } - } - - void *m_handle{nullptr}; string m_exec; - AspellSpeller *m_speller{nullptr}; + ExecCmd m_speller; #ifdef _WIN32 string m_datadir; #endif @@ -118,7 +55,6 @@ bool Aspell::init(string &reason) { - std::unique_lock locker(o_aapi_mutex); deleteZ(m_data); // Language: we get this from the configuration, else from the NLS @@ -149,11 +85,10 @@ m_config->getConfParam("aspellAddCreateParam", m_data->m_addCreateParam); #ifdef _WIN32 m_data->m_datadir = path_cat( - path_pkgdatadir(), - "filters/aspell-installed/mingw32/lib/aspell-0.60"); + path_pkgdatadir(), "filters/aspell-installed/mingw32/lib/aspell-0.60"); if (m_data->m_addCreateParam.empty()) { - m_data->m_addCreateParam = string("--local-data-dir=") + - path_cat(m_config->getConfDir(), "aspell"); + m_data->m_addCreateParam = + string("--local-data-dir=") + path_cat(m_config->getConfDir(), "aspell"); } #endif // WIN32 @@ -179,110 +114,16 @@ return false; } - // Don't know what with Apple and (DY)LD_LIBRARY_PATH. Does not work - // So we look in all ../lib in the PATH... -#if defined(__APPLE__) - vector path; - const char *pp = getenv("PATH"); - if (pp) { - stringToTokens(pp, path, ":"); - } -#endif - - reason = "Could not open shared library "; - string libbase("libaspell"); - string lib; - for (const auto& suff : aspell_lib_suffixes) { - lib = libbase + suff; - reason += string("[") + lib + "] "; - if ((m_data->m_handle = dlib_open(lib)) != 0) { - reason.erase(); - goto found; - } - // Above was the normal lookup: let dlopen search the directories. - // Also look in other places for Apple and Windows. -#if defined(__APPLE__) - for (const auto& dir : path) { - string lib1 = path_canon(dir + "/../lib/" + lib); - if ((m_data->m_handle = dlib_open(lib1)) != 0) { - reason.erase(); - lib=lib1; - goto found; - } - } -#endif -#if defined(_WIN32) && !defined(_MSC_VER) - // Look in the directory of the aspell binary. When building - // with msvc, the aspell .exe is still the mingw one, but we - // copy the msvc dll in the recoll top directory, so no need - // to look in the aspell one. - { - string bindir = path_getfather(m_data->m_exec); - string lib1 = path_cat(bindir, lib); - if ((m_data->m_handle = dlib_open(lib1)) != 0) { - reason.erase(); - lib=lib1; - goto found; - } - } -#endif - } - -found: - if (m_data->m_handle == 0) { - reason += string(" : ") + dlib_error(); - deleteZ(m_data); - return false; - } - - string badnames; - NMTOPTR(new_aspell_config, (struct AspellConfig *(*)())); - NMTOPTR(aspell_config_replace, (int (*)(struct AspellConfig *, - const char *, const char *))); - NMTOPTR(new_aspell_speller, - (struct AspellCanHaveError *(*)(struct AspellConfig *))); - NMTOPTR(delete_aspell_config, - (void (*)(struct AspellConfig *))); - NMTOPTR(delete_aspell_can_have_error, - (void (*)(struct AspellCanHaveError *))); - NMTOPTR(to_aspell_speller, - (struct AspellSpeller *(*)(struct AspellCanHaveError *))); - NMTOPTR(aspell_speller_config, - (struct AspellConfig *(*)(struct AspellSpeller *))); - NMTOPTR(aspell_speller_suggest, - (const struct AspellWordList *(*)(struct AspellSpeller *, - const char *, int))); - NMTOPTR(aspell_speller_check, - (int (*)(struct AspellSpeller *, const char *, int))); - NMTOPTR(aspell_word_list_elements, - (struct AspellStringEnumeration *(*) - (const struct AspellWordList *))); - NMTOPTR(aspell_string_enumeration_next, - (const char * (*)(struct AspellStringEnumeration *))); - NMTOPTR(delete_aspell_string_enumeration, - (void (*)(struct AspellStringEnumeration *))); - NMTOPTR(aspell_error, - (const struct AspellError*(*)(const struct AspellCanHaveError *))); - NMTOPTR(aspell_error_message, - (const char *(*)(const struct AspellCanHaveError *))); - NMTOPTR(aspell_speller_error_message, - (const char *(*)(const struct AspellSpeller *))); - NMTOPTR(delete_aspell_speller, (void (*)(struct AspellSpeller *))); - - if (!badnames.empty()) { - reason = string("Aspell::init: symbols not found:") + badnames; - deleteZ(m_data); - return false; - } - return true; } + bool Aspell::ok() const { - return m_data != 0 && m_data->m_handle != 0; + return nullptr != m_data; } + string Aspell::dicPath() { string ccdir = m_config->getAspellcacheDir(); @@ -409,84 +250,66 @@ return true; } -static const unsigned int ldatadiroptsz = - string("--local-data-dir=").size(); - bool Aspell::make_speller(string& reason) { if (!ok()) return false; - if (m_data->m_speller != 0) + if (m_data->m_speller.getChildPid() > 0) return true; - AspellCanHaveError *ret; + // aspell --lang=[lang] --encoding=utf-8 --master=[dicPath()] --sug-mode=fast --mode=none pipe + + string cmdstring(m_data->m_exec); + + ExecCmd aspell; + vector args; + + args.push_back(string("--lang=")+ m_lang); + cmdstring += string(" ") + args.back(); + + args.push_back("--encoding=utf-8"); + cmdstring += string(" ") + args.back(); - AspellConfig *config = aapi.new_aspell_config(); - aapi.aspell_config_replace(config, "lang", m_lang.c_str()); - aapi.aspell_config_replace(config, "encoding", "utf-8"); - aapi.aspell_config_replace(config, "master", dicPath().c_str()); - aapi.aspell_config_replace(config, "sug-mode", "fast"); #ifdef _WIN32 - aapi.aspell_config_replace(config, "data-dir", m_data->m_datadir.c_str()); + args.push_back(string("--data-dir=") + m_data->m_datadir); + cmdstring += string(" ") + args.back(); #endif - if (m_data->m_addCreateParam.size() > ldatadiroptsz) { - aapi.aspell_config_replace( - config, "local-data-dir", - m_data->m_addCreateParam.substr(ldatadiroptsz).c_str()); - } - // aapi.aspell_config_replace(config, "sug-edit-dist", "2"); - ret = aapi.new_aspell_speller(config); - aapi.delete_aspell_config(config); - if (aapi.aspell_error(ret) != 0) { - reason = aapi.aspell_error_message(ret); - aapi.delete_aspell_can_have_error(ret); - return false; + if (!m_data->m_addCreateParam.empty()) { + args.push_back(m_data->m_addCreateParam); + cmdstring += string(" ") + args.back(); } - m_data->m_speller = aapi.to_aspell_speller(ret); - return true; -} -bool Aspell::check(const string &iterm, string& reason) -{ - LOGDEB("Aspell::check [" << iterm << "]\n"); - string mterm(iterm); + args.push_back(string("--master=") + dicPath()); + cmdstring += string(" ") + args.back(); - if (!Rcl::Db::isSpellingCandidate(mterm)) { - LOGDEB0("Aspell::check: [" << mterm << - " not spelling candidate, return true\n"); - return true; - } - if (!ok() || !make_speller(reason)) - return false; - if (iterm.empty()) - return true; //?? + args.push_back(string("--sug-mode=fast")); + cmdstring += string(" ") + args.back(); - if (!o_index_stripchars) { - string lower; - if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) { - LOGERR("Aspell::check: cant lowercase input\n"); - return false; - } - mterm.swap(lower); - } + args.push_back(string("--mode=none")); + cmdstring += string(" ") + args.back(); - int ret = aapi.aspell_speller_check(m_data->m_speller, - mterm.c_str(), mterm.length()); - reason.clear(); - switch (ret) { - case 0: return false; - case 1: return true; - default: - case -1: - reason.append("Aspell error: "); - reason.append(aapi.aspell_speller_error_message(m_data->m_speller)); + args.push_back("pipe"); + cmdstring += string(" ") + args.back(); + + LOGDEB("Starting aspell command [" << cmdstring << "]\n"); + if (m_data->m_speller.startExec(m_data->m_exec, args, true, true) != 0) { + reason += "Can't start aspell: " + cmdstring; return false; } + // Read initial line from aspell: version etc. + string line; + if (m_data->m_speller.getline(line, 2) <= 0) { + reason += "Aspell: failed reading initial line"; + m_data->m_speller.zapChild(); + return false; + } + LOGDEB("rclaspell: aspell initial answer: [" << line << "]\n"); + return true; } -bool Aspell::suggest(Rcl::Db &db, const string &_term, - list& suggestions, string& reason) +bool Aspell::suggest( + Rcl::Db &db, const string &_term, vector& suggestions, string& reason) { LOGDEB("Aspell::suggest: term [" << _term << "]\n"); if (!ok() || !make_speller(reason)) @@ -496,8 +319,7 @@ return true; //?? if (!Rcl::Db::isSpellingCandidate(mterm)) { - LOGDEB0("Aspell::suggest: [" << mterm << - " not spelling candidate, return empty/true\n"); + LOGDEB0("Aspell::suggest: [" << mterm << " not spelling candidate, return empty/true\n"); return true; } @@ -510,27 +332,37 @@ mterm.swap(lower); } - const AspellWordList *wl = - aapi.aspell_speller_suggest(m_data->m_speller, - mterm.c_str(), mterm.length()); - if (wl == 0) { - reason = aapi.aspell_speller_error_message(m_data->m_speller); - return false; - } - AspellStringEnumeration *els = aapi.aspell_word_list_elements(wl); - const char *word; - while ((word = aapi.aspell_string_enumeration_next(els)) != 0) { - LOGDEB0("Aspell::suggest: got [" << word << "]\n"); - // Check that the word exists in the index (we don't want - // aspell computed stuff, only exact terms from the - // dictionary). We used to also check that it stems - // differently from the base word but this is complicated - // (stemming on/off + language), so we now leave this to the - // caller. + m_data->m_speller.send(mterm + "\n"); + std::string line; + if (m_data->m_speller.getline(line, 3) <= 0) { + reason.append("Aspell error: "); + return false; + } + LOGDEB1("ASPELL: got answer: " << line << "\n"); + string empty; + if (m_data->m_speller.getline(empty, 1) <= 0) { + reason.append("Aspell: failed reading final empty line\n"); + return false; + } + + if (line[0] == '*' || line[0] == '#') { + // Word is in dictionary, or there are no suggestions + return true; + } + string::size_type colon; + // Aspell suggestions line: & original count offset: miss, miss, … + if (line[0] != '&' || (colon = line.find(':')) == string::npos || colon == line.size()-1) { + // ?? + reason.append("Aspell: bad answer line: "); + reason.append(line); + return false; + } + std::vector words; + stringSplitString(line.substr(colon + 2), words, ", "); + for (const auto& word : words) { if (db.termExists(word)) suggestions.push_back(word); } - aapi.delete_aspell_string_enumeration(els); return true; } diff -Nru kio-recoll-1.30.1/aspell/rclaspell.h kio-recoll-1.31.0/aspell/rclaspell.h --- kio-recoll-1.30.1/aspell/rclaspell.h 2020-09-05 07:43:16.000000000 +0000 +++ kio-recoll-1.31.0/aspell/rclaspell.h 2021-04-23 08:38:13.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2006 J.F.Dockes +/* Copyright (C) 2006-2021 J.F.Dockes * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -27,12 +27,10 @@ * exist in the document set for a given word. * A specific aspell dictionary is created out of all the terms in the * xapian index, and we then use it to expand a term to spelling neighbours. - * We use the aspell C api for term expansion, but have - * to execute the program to create dictionaries. */ #include -#include +#include #include "rclconfig.h" #include "rcldb.h" @@ -40,7 +38,7 @@ class AspellData; class Aspell { - public: +public: Aspell(const RclConfig *cnf); ~Aspell(); @@ -54,19 +52,11 @@ * of an indexing pass. */ bool buildDict(Rcl::Db &db, std::string &reason); - /** Check that word is in dictionary. Note that this would mean - * that the EXACT word is: aspell just does a lookup, no - * grammatical, case or diacritics magic of any kind - * - * @return true if word in dic, false if not. reason.size() -> error - */ - bool check(const std::string& term, std::string& reason); - /** Return a list of possible expansions for a given word */ bool suggest(Rcl::Db &db, const std::string& term, - std::list &suggestions, std::string &reason); - - private: + std::vector &suggestions, std::string &reason); + +private: std::string dicPath(); const RclConfig *m_config; std::string m_lang; diff -Nru kio-recoll-1.30.1/common/autoconfig.h.in kio-recoll-1.31.0/common/autoconfig.h.in --- kio-recoll-1.30.1/common/autoconfig.h.in 2021-03-01 09:55:07.000000000 +0000 +++ kio-recoll-1.31.0/common/autoconfig.h.in 2021-04-23 08:53:57.000000000 +0000 @@ -3,9 +3,6 @@ /* Define if building universal (internal helper macro) */ #undef AC_APPLE_UNIVERSAL_BUILD -/* Path to the aspell api include file */ -#undef ASPELL_INCLUDE - /* Path to the aspell program */ #undef ASPELL_PROG diff -Nru kio-recoll-1.30.1/configure kio-recoll-1.31.0/configure --- kio-recoll-1.30.1/configure 2021-04-12 13:01:47.000000000 +0000 +++ kio-recoll-1.31.0/configure 2021-04-27 08:13:47.000000000 +0000 @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Recoll 1.30.1. +# Generated by GNU Autoconf 2.69 for Recoll 1.31.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -587,8 +587,8 @@ # Identity of this package. PACKAGE_NAME='Recoll' PACKAGE_TARNAME='recoll' -PACKAGE_VERSION='1.30.1' -PACKAGE_STRING='Recoll 1.30.1' +PACKAGE_VERSION='1.31.0' +PACKAGE_STRING='Recoll 1.31.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1418,7 +1418,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Recoll 1.30.1 to adapt to many kinds of systems. +\`configure' configures Recoll 1.31.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1493,7 +1493,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Recoll 1.30.1:";; + short | recursive ) echo "Configuration of Recoll 1.31.0:";; esac cat <<\_ACEOF @@ -1667,7 +1667,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Recoll configure 1.30.1 +Recoll configure 1.31.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2153,7 +2153,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Recoll $as_me 1.30.1, which was +It was created by Recoll $as_me 1.31.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3022,7 +3022,7 @@ # Define the identity of the package. PACKAGE='recoll' - VERSION='1.30.1' + VERSION='1.31.0' # Some tools Automake needs. @@ -17659,24 +17659,6 @@ #define ASPELL_PROG "$aspellProg" _ACEOF - if test -f $aspellBase/include/aspell.h ; then - -cat >>confdefs.h <<_ACEOF -#define ASPELL_INCLUDE "$aspellBase/include/aspell.h" -_ACEOF - - else - { $as_echo "$as_me:${as_lineno-$LINENO}: aspell support enabled but aspell package not found. Compiling with internal aspell interface file" >&5 -$as_echo "$as_me: aspell support enabled but aspell package not found. Compiling with internal aspell interface file" >&6;} - $as_echo "#define ASPELL_INCLUDE \"aspell-local.h\"" >>confdefs.h - - fi - else - # aspell support enabled but no aspell install yet - { $as_echo "$as_me:${as_lineno-$LINENO}: aspell support enabled but aspell package not found. Compiling with internal aspell interface file" >&5 -$as_echo "$as_me: aspell support enabled but aspell package not found. Compiling with internal aspell interface file" >&6;} - $as_echo "#define ASPELL_INCLUDE \"aspell-local.h\"" >>confdefs.h - fi fi @@ -20067,7 +20049,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Recoll $as_me 1.30.1, which was +This file was extended by Recoll $as_me 1.31.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -20133,7 +20115,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Recoll config.status 1.30.1 +Recoll config.status 1.31.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff -Nru kio-recoll-1.30.1/configure.ac kio-recoll-1.31.0/configure.ac --- kio-recoll-1.30.1/configure.ac 2021-02-26 08:43:17.000000000 +0000 +++ kio-recoll-1.31.0/configure.ac 2021-04-23 08:53:21.000000000 +0000 @@ -114,20 +114,8 @@ if test X$aspellProg != X ; then aspellBase=`dirname $aspellProg` aspellBase=`dirname $aspellBase` - AC_DEFINE_UNQUOTED(ASPELL_PROG, "$aspellProg", - [Path to the aspell program]) - if test -f $aspellBase/include/aspell.h ; then - AC_DEFINE_UNQUOTED(ASPELL_INCLUDE, "$aspellBase/include/aspell.h", - [Path to the aspell api include file]) - else - AC_MSG_NOTICE([aspell support enabled but aspell package not found. Compiling with internal aspell interface file]) - AC_DEFINE(ASPELL_INCLUDE, ["aspell-local.h"]) - fi - else - # aspell support enabled but no aspell install yet - AC_MSG_NOTICE([aspell support enabled but aspell package not found. Compiling with internal aspell interface file]) - AC_DEFINE(ASPELL_INCLUDE, ["aspell-local.h"]) - fi + AC_DEFINE_UNQUOTED(ASPELL_PROG, "$aspellProg", [Path to the aspell program]) + fi fi if test -f /usr/include/sys/inotify.h -o -f /usr/include/linux/inotify.h; then diff -Nru kio-recoll-1.30.1/debian/changelog kio-recoll-1.31.0/debian/changelog --- kio-recoll-1.30.1/debian/changelog 2021-04-12 14:10:00.000000000 +0000 +++ kio-recoll-1.31.0/debian/changelog 2021-04-27 09:19:00.000000000 +0000 @@ -1,3 +1,9 @@ +kio-recoll (1.31.0-1~ppa1~bionic1) bionic; urgency=low + + * Follow recoll version + + -- Jean-Francois Dockes Tue, 27 Apr 2021 10:19:00 +0100 + kio-recoll (1.30.1-1~ppa1~bionic1) bionic; urgency=low * Follow recoll version diff -Nru kio-recoll-1.30.1/filters/rclpython.py kio-recoll-1.31.0/filters/rclpython.py --- kio-recoll-1.30.1/filters/rclpython.py 2021-03-29 07:48:31.000000000 +0000 +++ kio-recoll-1.31.0/filters/rclpython.py 2021-04-14 12:23:57.000000000 +0000 @@ -223,18 +223,6 @@ sourcefile.close() -def _htmlwrapplain(txt, title=b"", charset=b"utf-8"): - return \ - b'\n\n' + \ - title + \ - b'\n' + \ - b'\n' + \ - b'\n
\n' + \
-        rclexecm.htmlescape(txt) + \
-        b'
\n\n\n' - class PythonDump(RclBaseHandler): def __init__(self, em): super(PythonDump, self).__init__(em) @@ -246,7 +234,8 @@ colorize_file(fn, out) return out.getvalue() else: - return _htmlwrapplain(open(fn, 'rb').read()) + self.outputmimetype = "text/plain" + return open(fn, 'rb').read() if __name__ == '__main__': proto = rclexecm.RclExecM() diff -Nru kio-recoll-1.30.1/index/fsfetcher.cpp kio-recoll-1.31.0/index/fsfetcher.cpp --- kio-recoll-1.30.1/index/fsfetcher.cpp 2020-09-05 07:43:16.000000000 +0000 +++ kio-recoll-1.31.0/index/fsfetcher.cpp 2021-04-16 09:23:47.000000000 +0000 @@ -57,14 +57,20 @@ out.data = fn; return true; } - + +void fsmakesig(const struct PathStat *stp, string& out) +{ + out = lltodecstr(stp->pst_size) + + lltodecstr(o_uptodate_test_use_mtime ? stp->pst_mtime : stp->pst_ctime); +} + bool FSDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig) { string fn; struct PathStat st; if (urltopath(cnf, idoc, fn, st) != DocFetcher::FetchOk) return false; - FsIndexer::makesig(&st, sig); + fsmakesig(&st, sig); return true; } diff -Nru kio-recoll-1.30.1/index/fsfetcher.h kio-recoll-1.31.0/index/fsfetcher.h --- kio-recoll-1.30.1/index/fsfetcher.h 2020-08-20 18:59:57.000000000 +0000 +++ kio-recoll-1.31.0/index/fsfetcher.h 2021-04-16 09:22:02.000000000 +0000 @@ -18,6 +18,7 @@ #define _FSFETCHER_H_INCLUDED_ #include "fetcher.h" +#include "pathut.h" /** * The file-system fetcher: @@ -32,4 +33,6 @@ virtual ~FSDocFetcher() {} }; +extern void fsmakesig(const struct PathStat *stp, std::string& out); + #endif /* _FSFETCHER_H_INCLUDED_ */ diff -Nru kio-recoll-1.30.1/index/fsindexer.cpp kio-recoll-1.31.0/index/fsindexer.cpp --- kio-recoll-1.30.1/index/fsindexer.cpp 2021-04-01 06:13:37.000000000 +0000 +++ kio-recoll-1.31.0/index/fsindexer.cpp 2021-04-21 10:51:55.000000000 +0000 @@ -48,6 +48,7 @@ #include "extrameta.h" #include "utf8fn.h" #include "idxdiags.h" +#include "fsfetcher.h" #if defined(HAVE_POSIX_FADVISE) #include #include @@ -103,8 +104,8 @@ } }; -FsIndexer::FsIndexer(RclConfig *cnf, Rcl::Db *db, DbIxStatusUpdater *updfunc) - : m_config(cnf), m_db(db), m_updater(updfunc), +FsIndexer::FsIndexer(RclConfig *cnf, Rcl::Db *db) + : m_config(cnf), m_db(db), m_missing(new FSIFIMissingStore), m_detectxattronly(false), m_noretryfailed(false) #ifdef IDX_THREADS @@ -186,12 +187,7 @@ if (!init()) return false; - if (m_updater) { -#ifdef IDX_THREADS - std::unique_lock locker(m_updater->m_mutex); -#endif - m_updater->status.dbtotdocs = m_db->docCnt(); - } + statusUpdater()->setDbTotDocs(m_db->docCnt()); m_walker.setSkippedPaths(m_config->getSkippedPaths()); if (quickshallow) { @@ -273,7 +269,7 @@ string topdir; for (;;) { // Used to test not root here, but root may be in topdirs ! - for (const auto tdlent : tdl) { + for (const auto& tdlent : tdl) { // the topdirs members are already canonized. LOGDEB1("matchesSkipped: comparing ancestor [" << mpath << "] to topdir [" << tdlent << "]\n"); @@ -512,12 +508,6 @@ } } -void FsIndexer::makesig(const struct PathStat *stp, string& out) -{ - out = lltodecstr(stp->pst_size) + - lltodecstr(o_uptodate_test_use_mtime ? stp->pst_mtime : stp->pst_ctime); -} - #ifdef IDX_THREADS // Called updworker as seen from here, but the first step (and only in // most meaningful configurations) is doing the word-splitting, which @@ -586,13 +576,8 @@ FsTreeWalker::Status FsIndexer::processone( const std::string &fn, const struct PathStat *stp, FsTreeWalker::CbFlag flg) { - if (m_updater) { -#ifdef IDX_THREADS - std::unique_lock locker(m_updater->m_mutex); -#endif - if (!m_updater->update()) { - return FsTreeWalker::FtwStop; - } + if (!statusUpdater()->update(DbIxStatus::DBIXS_FILES, fn)) { + return FsTreeWalker::FtwStop; } // If we're changing directories, possibly adjust parameters (set @@ -665,7 +650,7 @@ // m/ctime and size and the possibly new value is checked against // the stored one. string sig; - makesig(stp, sig); + fsmakesig(stp, sig); string udi; make_udi(fn, cstr_null, udi); unsigned int existingDoc; @@ -706,16 +691,9 @@ if (!needupdate) { LOGDEB0("processone: up to date: " << fn << "\n"); - if (m_updater) { -#ifdef IDX_THREADS - std::unique_lock locker(m_updater->m_mutex); -#endif - // Status bar update, abort request etc. - m_updater->status.fn = fn; - ++(m_updater->status.filesdone); - if (!m_updater->update()) { - return FsTreeWalker::FtwStop; - } + if (!statusUpdater()->update( + DbIxStatus::DBIXS_FILES, fn, DbIxStatusUpdater::IncrFilesDone)) { + return FsTreeWalker::FtwStop; } return FsTreeWalker::FtwOk; } @@ -827,26 +805,20 @@ } // Tell what we are doing and check for interrupt request - if (m_updater) { -#ifdef IDX_THREADS - std::unique_lock locker(m_updater->m_mutex); -#endif - ++(m_updater->status.docsdone); - if (m_updater->status.dbtotdocs < m_updater->status.docsdone) - m_updater->status.dbtotdocs = m_updater->status.docsdone; - m_updater->status.fn = fn; - if (!doc.ipath.empty()) { - m_updater->status.fn += "|" + doc.ipath; - } else { - if (fis == FileInterner::FIError) { - ++(m_updater->status.fileerrors); - } - ++(m_updater->status.filesdone); - } - if (!m_updater->update()) { - return FsTreeWalker::FtwStop; + int incr = DbIxStatusUpdater::IncrDocsDone; + std::string sfn(fn); + if (!doc.ipath.empty()) { + sfn += "|" + doc.ipath; + } else { + if (fis == FileInterner::FIError) { + incr |= DbIxStatusUpdater::IncrFileErrors; } + incr |= DbIxStatusUpdater::IncrFilesDone; } + if (!statusUpdater()->update(DbIxStatus::DBIXS_FILES, sfn, incr)) { + return FsTreeWalker::FtwStop; + } + } if (fis == FileInterner::FIError) { diff -Nru kio-recoll-1.30.1/index/fsindexer.h kio-recoll-1.31.0/index/fsindexer.h --- kio-recoll-1.30.1/index/fsindexer.h 2021-02-26 08:43:17.000000000 +0000 +++ kio-recoll-1.31.0/index/fsindexer.h 2021-04-16 09:22:14.000000000 +0000 @@ -26,7 +26,6 @@ #include "workqueue.h" #endif // IDX_THREADS -class DbIxStatusUpdater; class FIMissingStore; struct PathStat; @@ -53,9 +52,8 @@ /** Constructor does nothing but store parameters * * @param cnf Configuration data - * @param updfunc Status updater callback */ - FsIndexer(RclConfig *cnf, Rcl::Db *db, DbIxStatusUpdater *updfunc = 0); + FsIndexer(RclConfig *cnf, Rcl::Db *db); virtual ~FsIndexer(); /** @@ -77,10 +75,6 @@ FsTreeWalker::Status processone(const string &fn, const struct PathStat *, FsTreeWalker::CbFlag); - /** Make signature for file up to date checks */ - static void makesig(const struct PathStat *stp, string& out); - - private: class PurgeCandidateRecorder { @@ -118,7 +112,6 @@ RclConfig *m_config; Rcl::Db *m_db; string m_reason; - DbIxStatusUpdater *m_updater; // Top/start directories list std::vector m_tdl; // Store for missing filters and associated mime types diff -Nru kio-recoll-1.30.1/index/idxstatus.cpp kio-recoll-1.31.0/index/idxstatus.cpp --- kio-recoll-1.30.1/index/idxstatus.cpp 2020-09-05 07:43:16.000000000 +0000 +++ kio-recoll-1.31.0/index/idxstatus.cpp 2021-04-16 10:50:48.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2017-2018 J.F.Dockes +/* Copyright (C) 2017-2021 J.F.Dockes * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -16,10 +16,18 @@ */ #include "autoconfig.h" -#include "idxstatus.h" +#include -#include "rclconfig.h" +#include "chrono.h" #include "conftree.h" +#include "idxstatus.h" +#include "log.h" +#include "rclconfig.h" +#include "x11mon.h" + +// Global stop request flag. This is checked in a number of place in the +// indexing routines. +int stopindexing; void readIdxStatus(RclConfig *config, DbIxStatus &status) { @@ -33,3 +41,126 @@ status.totfiles = (int)cs.getInt("totfiles", 0); status.hasmonitor = cs.getBool("hasmonitor", false); } + +// Receive status updates from the ongoing indexing operation +// Also check for an interrupt request and return the info to caller which +// should subsequently orderly terminate what it is doing. +class DbIxStatusUpdater::Internal { +public: +#ifdef IDX_THREADS + std::mutex m_mutex; +#endif + Internal(const RclConfig *config, bool nox11mon) + : m_file(config->getIdxStatusFile().c_str()), m_stopfilename(config->getIdxStopFile()), + nox11monitor(nox11mon) { + // The total number of files included in the index is actually + // difficult to compute from the index itself. For display + // purposes, we save it in the status file from indexing to + // indexing (mostly...) + string stf; + if (m_file.get("totfiles", stf)) { + status.totfiles = atoi(stf.c_str()); + } + } + + virtual bool update() { + if (status.dbtotdocs < status.docsdone) + status.dbtotdocs = status.docsdone; + // Update the status file. Avoid doing it too often. Always do + // it at the end (status DONE) + if (status.phase == DbIxStatus::DBIXS_DONE || + status.phase != m_prevphase || m_chron.millis() > 300) { + if (status.totfiles < status.filesdone || status.phase == DbIxStatus::DBIXS_DONE) { + status.totfiles = status.filesdone; + } + m_prevphase = status.phase; + m_chron.restart(); + m_file.holdWrites(true); + m_file.set("phase", int(status.phase)); + m_file.set("docsdone", status.docsdone); + m_file.set("filesdone", status.filesdone); + m_file.set("fileerrors", status.fileerrors); + m_file.set("dbtotdocs", status.dbtotdocs); + m_file.set("totfiles", status.totfiles); + m_file.set("fn", status.fn); + m_file.set("hasmonitor", status.hasmonitor); + m_file.holdWrites(false); + } + if (path_exists(m_stopfilename)) { + LOGINF("recollindex: asking indexer to stop because " << m_stopfilename << " exists\n"); + path_unlink(m_stopfilename); + stopindexing = true; + } + if (stopindexing) { + return false; + } + +#ifndef DISABLE_X11MON + // If we are in the monitor, we also need to check X11 status + // during the initial indexing pass (else the user could log + // out and the indexing would go on, not good (ie: if the user + // logs in again, the new recollindex will fail). + if (status.hasmonitor && !nox11monitor && !x11IsAlive()) { + LOGDEB("X11 session went away during initial indexing pass\n"); + stopindexing = true; + return false; + } +#endif + return true; + } + + DbIxStatus status; + ConfSimple m_file; + string m_stopfilename; + Chrono m_chron; + bool nox11monitor{false}; + DbIxStatus::Phase m_prevphase{DbIxStatus::DBIXS_NONE}; +}; + + +DbIxStatusUpdater::DbIxStatusUpdater(const RclConfig *config, bool nox11monitor) { + m = new Internal(config, nox11monitor); +} + +void DbIxStatusUpdater::setMonitor(bool onoff) +{ + m->status.hasmonitor = onoff; +} + +void DbIxStatusUpdater::setDbTotDocs(int totdocs) +{ +#ifdef IDX_THREADS + std::unique_lock lock(m->m_mutex); +#endif + m->status.dbtotdocs = totdocs; +} + +bool DbIxStatusUpdater::update(DbIxStatus::Phase phase, const string& fn, int incr) { +#ifdef IDX_THREADS + std::unique_lock lock(m->m_mutex); +#endif + + // We don't change a FLUSH status except if the new status is NONE + // (recollindex init or rcldb after commit(). Else, the flush status maybe + // overwritten by a "file updated" status and not be displayed + if (phase == DbIxStatus::DBIXS_NONE || m->status.phase != DbIxStatus::DBIXS_FLUSH) + m->status.phase = phase; + m->status.fn = fn; + if (incr & IncrDocsDone) + m->status.docsdone++; + if (incr & IncrFilesDone) + m->status.filesdone++; + if (incr & IncrFileErrors) + m->status.fileerrors++; + return m->update(); +} + +static DbIxStatusUpdater *updater; + +DbIxStatusUpdater *statusUpdater(RclConfig *config, bool nox11mon) +{ + if (updater) { + return updater; + } + return (updater = new DbIxStatusUpdater(config, nox11mon)); +} diff -Nru kio-recoll-1.30.1/index/idxstatus.h kio-recoll-1.31.0/index/idxstatus.h --- kio-recoll-1.30.1/index/idxstatus.h 2020-09-05 07:43:16.000000000 +0000 +++ kio-recoll-1.31.0/index/idxstatus.h 2021-04-16 09:46:13.000000000 +0000 @@ -24,10 +24,8 @@ // $RECOLL_CONFDIR/idxstatus.txt class DbIxStatus { public: - enum Phase {DBIXS_NONE, - DBIXS_FILES, DBIXS_PURGE, DBIXS_STEMDB, DBIXS_CLOSING, - DBIXS_MONITOR, - DBIXS_DONE}; + enum Phase {DBIXS_NONE, DBIXS_FILES, DBIXS_FLUSH, DBIXS_PURGE, DBIXS_STEMDB, DBIXS_CLOSING, + DBIXS_MONITOR, DBIXS_DONE}; Phase phase; std::string fn; // Last file processed int docsdone; // Documents actually updated @@ -53,4 +51,27 @@ class RclConfig; extern void readIdxStatus(RclConfig *config, DbIxStatus &status); +/** Callback to say what we're doing. If the update func returns false, we + * stop as soon as possible without corrupting state */ +class DbIxStatusUpdater { +public: + DbIxStatusUpdater(const RclConfig *config, bool nox11monitor); + virtual ~DbIxStatusUpdater(){} + + enum Incr {IncrNone, IncrDocsDone = 0x1, IncrFilesDone = 0x2, IncrFileErrors = 0x4}; + // Change phase/fn and update + virtual bool update(DbIxStatus::Phase phase, const std::string& fn, int incr = IncrNone); + + void setMonitor(bool onoff); + void setDbTotDocs(int totdocs); + + class Internal; +private: + Internal *m; +}; + +// We use the updater as a singleton everywhere. It is instanciated in +// idxstatus.cpp. Must be called once with non-null config at first. +extern DbIxStatusUpdater *statusUpdater(RclConfig *config=nullptr, bool nox11monitor=false); + #endif /* _IDXSTATUS_H_INCLUDED_ */ diff -Nru kio-recoll-1.30.1/index/indexer.cpp kio-recoll-1.31.0/index/indexer.cpp --- kio-recoll-1.30.1/index/indexer.cpp 2020-12-14 10:52:59.000000000 +0000 +++ kio-recoll-1.31.0/index/indexer.cpp 2021-04-16 11:24:44.000000000 +0000 @@ -42,10 +42,6 @@ using std::string; using std::vector; -// Global stop request flag. This is checked in a number of place in the -// indexing routines. -int stopindexing; - // This would more logically live in recollindex.cpp, but then librecoll would // have an undefined symbol ConfSimple idxreasons; @@ -94,10 +90,8 @@ } #endif -ConfIndexer::ConfIndexer(RclConfig *cnf, DbIxStatusUpdater *updfunc) - : m_config(cnf), m_db(cnf), m_fsindexer(0), - m_doweb(false), m_webindexer(0), - m_updater(updfunc) +ConfIndexer::ConfIndexer(RclConfig *cnf) + : m_config(cnf), m_db(cnf) { m_config->getConfParam("processwebqueue", &m_doweb); } @@ -135,7 +129,7 @@ { LOGDEB("ConfIndexer::firstFsIndexingSequence\n"); deleteZ(m_fsindexer); - m_fsindexer = new FsIndexer(m_config, &m_db, m_updater); + m_fsindexer = new FsIndexer(m_config, &m_db); if (!m_fsindexer) { return false; } @@ -170,7 +164,7 @@ firstFsIndexingSequence(); } deleteZ(m_fsindexer); - m_fsindexer = new FsIndexer(m_config, &m_db, m_updater); + m_fsindexer = new FsIndexer(m_config, &m_db); if (!m_fsindexer || !m_fsindexer->index(flags)) { if (stopindexing) { addIdxReason("indexer", "Indexing was interrupted."); @@ -185,7 +179,7 @@ if (m_doweb && (typestorun & IxTWebQueue)) { runWebFilesMoverScript(m_config); deleteZ(m_webindexer); - m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater); + m_webindexer = new WebQueueIndexer(m_config, &m_db); if (!m_webindexer || !m_webindexer->index()) { m_db.close(); addIdxReason("indexer", "Web index creation failed. See" + logloc); @@ -196,7 +190,7 @@ if (typestorun == IxTAll) { // Get rid of all database entries that don't exist in the // filesystem anymore. Only if all *configured* indexers ran. - if (m_updater && !m_updater->update(DbIxStatus::DBIXS_PURGE, "")) { + if (!statusUpdater()->update(DbIxStatus::DBIXS_PURGE, string())) { m_db.close(); addIdxReason("indexer", "Index purge failed. See" + logloc); return false; @@ -207,8 +201,7 @@ // The close would be done in our destructor, but we want status // here. Makes no sense to check for cancel, we'll have to close // anyway - if (m_updater) - m_updater->update(DbIxStatus::DBIXS_CLOSING, string()); + statusUpdater()->update(DbIxStatus::DBIXS_CLOSING, string()); if (!m_db.close()) { LOGERR("ConfIndexer::index: error closing database in " << m_config->getDbDir() << "\n"); @@ -216,13 +209,13 @@ return false; } - if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string())) + if (!statusUpdater()->update(DbIxStatus::DBIXS_CLOSING, string())) return false; bool ret = true; if (!createStemmingDatabases()) { ret = false; } - if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string())) + if (!statusUpdater()->update(DbIxStatus::DBIXS_CLOSING, string())) return false; // Don't fail indexing because of an aspell issue: we ignore the status. @@ -230,8 +223,7 @@ (void)createAspellDict(); clearMimeHandlerCache(); - if (m_updater) - m_updater->update(DbIxStatus::DBIXS_DONE, string()); + statusUpdater()->update(DbIxStatus::DBIXS_DONE, string()); return ret; } @@ -252,7 +244,7 @@ m_config->setKeyDir(cstr_null); bool ret = false; if (!m_fsindexer) - m_fsindexer = new FsIndexer(m_config, &m_db, m_updater); + m_fsindexer = new FsIndexer(m_config, &m_db); if (m_fsindexer) ret = m_fsindexer->indexFiles(myfiles, flag); LOGDEB2("ConfIndexer::indexFiles: fsindexer returned " << ret << ", " << @@ -261,7 +253,7 @@ if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) { if (!m_webindexer) - m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater); + m_webindexer = new WebQueueIndexer(m_config, &m_db); if (m_webindexer) { ret = ret && m_webindexer->indexFiles(myfiles); } else { @@ -313,14 +305,14 @@ bool ret = false; m_config->setKeyDir(cstr_null); if (!m_fsindexer) - m_fsindexer = new FsIndexer(m_config, &m_db, m_updater); + m_fsindexer = new FsIndexer(m_config, &m_db); if (m_fsindexer) ret = m_fsindexer->purgeFiles(myfiles); #ifndef DISABLE_WEB_INDEXER if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) { if (!m_webindexer) - m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater); + m_webindexer = new WebQueueIndexer(m_config, &m_db); if (m_webindexer) { ret = ret && m_webindexer->purgeFiles(myfiles); } else { diff -Nru kio-recoll-1.30.1/index/indexer.h kio-recoll-1.31.0/index/indexer.h --- kio-recoll-1.30.1/index/indexer.h 2021-02-26 08:43:17.000000000 +0000 +++ kio-recoll-1.31.0/index/indexer.h 2021-04-16 08:18:39.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2004 J.F.Dockes +/* Copyright (C) 2004-2021 J.F.Dockes * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -16,7 +16,6 @@ */ #ifndef _INDEXER_H_INCLUDED_ #define _INDEXER_H_INCLUDED_ -#include "rclconfig.h" #include #include @@ -24,6 +23,7 @@ #include #include +#include "rclconfig.h" #include "rcldb.h" #include "rcldoc.h" #include "idxstatus.h" @@ -31,31 +31,6 @@ class FsIndexer; class WebQueueIndexer; -/** Callback to say what we're doing. If the update func returns false, we - * stop as soon as possible without corrupting state */ -class DbIxStatusUpdater { -public: -#ifdef IDX_THREADS - std::mutex m_mutex; -#endif - DbIxStatus status; - virtual ~DbIxStatusUpdater(){} - - // Convenience: change phase/fn and update - virtual bool update(DbIxStatus::Phase phase, const string& fn) - { -#ifdef IDX_THREADS - std::unique_lock lock(m_mutex); -#endif - status.phase = phase; - status.fn = fn; - return update(); - } - - // To be implemented by user for sending info somewhere - virtual bool update() = 0; -}; - /** * The top level batch indexing object. Processes the configuration, * then invokes file system walking or other to populate/update the @@ -64,7 +39,7 @@ class ConfIndexer { public: enum runStatus {IndexerOk, IndexerError}; - ConfIndexer(RclConfig *cnf, DbIxStatusUpdater *updfunc = 0); + ConfIndexer(RclConfig *cnf); virtual ~ConfIndexer(); // Indexer types. Maybe we'll have something more dynamic one day @@ -118,11 +93,10 @@ private: RclConfig *m_config; Rcl::Db m_db; - FsIndexer *m_fsindexer; - bool m_doweb; - WebQueueIndexer *m_webindexer; - DbIxStatusUpdater *m_updater; - string m_reason; + FsIndexer *m_fsindexer{nullptr}; + bool m_doweb{false}; + WebQueueIndexer *m_webindexer{nullptr}; + string m_reason; // The first time we index, we do things a bit differently to // avoid user frustration (make at least some results available diff -Nru kio-recoll-1.30.1/index/recollindex.cpp kio-recoll-1.31.0/index/recollindex.cpp --- kio-recoll-1.30.1/index/recollindex.cpp 2021-04-01 09:07:38.000000000 +0000 +++ kio-recoll-1.31.0/index/recollindex.cpp 2021-04-16 11:29:45.000000000 +0000 @@ -60,7 +60,6 @@ #endif #include "execmd.h" #include "checkretryfailed.h" -#include "idxstatus.h" #include "circache.h" #include "idxdiags.h" @@ -117,79 +116,6 @@ recoll_exitready(); } -// Receive status updates from the ongoing indexing operation -// Also check for an interrupt request and return the info to caller which -// should subsequently orderly terminate what it is doing. -class MyUpdater : public DbIxStatusUpdater { -public: - MyUpdater(const RclConfig *config) - : m_file(config->getIdxStatusFile().c_str()), - m_stopfilename(config->getIdxStopFile()), - m_prevphase(DbIxStatus::DBIXS_NONE) { - // The total number of files included in the index is actually - // difficult to compute from the index itself. For display - // purposes, we save it in the status file from indexing to - // indexing (mostly...) - string stf; - if (m_file.get("totfiles", stf)) { - status.totfiles = atoi(stf.c_str()); - } - } - - virtual bool update() { - // Update the status file. Avoid doing it too often. Always do - // it at the end (status DONE) - if (status.phase == DbIxStatus::DBIXS_DONE || - status.phase != m_prevphase || m_chron.millis() > 300) { - if (status.totfiles < status.filesdone || - status.phase == DbIxStatus::DBIXS_DONE) { - status.totfiles = status.filesdone; - } - m_prevphase = status.phase; - m_chron.restart(); - m_file.holdWrites(true); - m_file.set("phase", int(status.phase)); - m_file.set("docsdone", status.docsdone); - m_file.set("filesdone", status.filesdone); - m_file.set("fileerrors", status.fileerrors); - m_file.set("dbtotdocs", status.dbtotdocs); - m_file.set("totfiles", status.totfiles); - m_file.set("fn", status.fn); - m_file.set("hasmonitor", status.hasmonitor); - m_file.holdWrites(false); - } - if (path_exists(m_stopfilename)) { - LOGINF("recollindex: asking indexer to stop because " << - m_stopfilename << " exists\n"); - path_unlink(m_stopfilename); - stopindexing = true; - } - if (stopindexing) { - return false; - } - -#ifndef DISABLE_X11MON - // If we are in the monitor, we also need to check X11 status - // during the initial indexing pass (else the user could log - // out and the indexing would go on, not good (ie: if the user - // logs in again, the new recollindex will fail). - if ((op_flags & OPT_m) && !(op_flags & OPT_x) && !x11IsAlive()) { - LOGDEB("X11 session went away during initial indexing pass\n"); - stopindexing = true; - return false; - } -#endif - return true; - } - -private: - ConfSimple m_file; - string m_stopfilename; - Chrono m_chron; - DbIxStatus::Phase m_prevphase; -}; -static MyUpdater *updater; - // This holds the state of topdirs (exist+nonempty) on indexing // startup. If it changes after a resume from sleep we interrupt the // indexing (the assumption being that a volume has been mounted or @@ -233,7 +159,7 @@ static void makeIndexerOrExit(RclConfig *config, bool inPlaceReset) { if (!confindexer) { - confindexer = new ConfIndexer(config, updater); + confindexer = new ConfIndexer(config); if (inPlaceReset) confindexer->setInPlaceReset(); } @@ -831,13 +757,20 @@ #endif Pidfile pidfile(config->getPidfile()); - updater = new MyUpdater(config); lockorexit(&pidfile, config); // Log something at LOGINFO to reset the trace file. Else at level // 3 it's not even truncated if all docs are up to date. LOGINFO("recollindex: starting up\n"); setMyPriority(config); + + // Init status updater + if (nullptr == statusUpdater(config, op_flags & OPT_x)) { + std::cerr << "Could not initialize status updater\n"; + LOGERR("Could not initialize status updater\n"); + exit(1); + } + statusUpdater()->update(DbIxStatus::DBIXS_NONE, ""); if (op_flags & OPT_r) { if (aremain != 1) @@ -899,9 +832,7 @@ } else if (op_flags & OPT_m) { if (aremain != 0) Usage(); - if (updater) { - updater->status.hasmonitor = true; - } + statusUpdater()->setMonitor(true); if (!(op_flags&OPT_D)) { LOGDEB("recollindex: daemonizing\n"); #ifndef _WIN32 @@ -960,11 +891,8 @@ #endif } - if (updater) { - updater->status.phase = DbIxStatus::DBIXS_MONITOR; - updater->status.fn.clear(); - updater->update(); - } + statusUpdater()->update(DbIxStatus::DBIXS_MONITOR, ""); + int opts = RCLMON_NONE; if (op_flags & OPT_D) opts |= RCLMON_NOFORK; @@ -991,11 +919,7 @@ addIdxReason("indexer", confindexer->getReason()); cerr << confindexer->getReason() << endl; } - if (updater) { - updater->status.phase = DbIxStatus::DBIXS_DONE; - updater->status.fn.clear(); - updater->update(); - } + statusUpdater()->update(DbIxStatus::DBIXS_DONE, ""); flushIdxReasons(); return !status; } diff -Nru kio-recoll-1.30.1/index/webqueue.cpp kio-recoll-1.31.0/index/webqueue.cpp --- kio-recoll-1.30.1/index/webqueue.cpp 2021-04-01 13:56:41.000000000 +0000 +++ kio-recoll-1.31.0/index/webqueue.cpp 2021-04-16 08:39:43.000000000 +0000 @@ -173,8 +173,8 @@ // Initialize. Compute paths and create a temporary directory that will be // used by internfile() -WebQueueIndexer::WebQueueIndexer(RclConfig *cnf, Rcl::Db *db, DbIxStatusUpdater *updfunc) - : m_config(cnf), m_db(db), m_updater(updfunc) +WebQueueIndexer::WebQueueIndexer(RclConfig *cnf, Rcl::Db *db) + : m_config(cnf), m_db(db) { m_queuedir = m_config->getWebQueueDir(); path_catslash(m_queuedir); @@ -255,13 +255,7 @@ void WebQueueIndexer::updstatus(const string& udi) { - if (m_updater) { - ++(m_updater->status.docsdone); - if (m_updater->status.dbtotdocs < m_updater->status.docsdone) - m_updater->status.dbtotdocs = m_updater->status.docsdone; - m_updater->status.fn = udi; - m_updater->update(); - } + statusUpdater()->update(DbIxStatus::DBIXS_FILES, udi, DbIxStatusUpdater::IncrDocsDone); } bool WebQueueIndexer::index() diff -Nru kio-recoll-1.30.1/index/webqueue.h kio-recoll-1.31.0/index/webqueue.h --- kio-recoll-1.30.1/index/webqueue.h 2021-03-24 09:36:00.000000000 +0000 +++ kio-recoll-1.31.0/index/webqueue.h 2021-04-16 08:26:57.000000000 +0000 @@ -30,7 +30,6 @@ #include "fstreewalk.h" #include "rcldoc.h" -class DbIxStatusUpdater; class CirCache; class RclConfig; class WebStore; @@ -40,8 +39,7 @@ class WebQueueIndexer : public FsTreeWalkerCB { public: - WebQueueIndexer(RclConfig *cnf, Rcl::Db *db, - DbIxStatusUpdater *updfunc = 0); + WebQueueIndexer(RclConfig *cnf, Rcl::Db *db); ~WebQueueIndexer(); /** This is called by the top indexer in recollindex. @@ -69,7 +67,6 @@ Rcl::Db *m_db{nullptr}; WebStore *m_cache{nullptr}; std::string m_queuedir; - DbIxStatusUpdater *m_updater{nullptr}; // Don't process the cache. Set by indexFiles(). bool m_nocacheindex{false}; // Config: page erase interval. We normally keep only one diff -Nru kio-recoll-1.30.1/Makefile.am kio-recoll-1.31.0/Makefile.am --- kio-recoll-1.30.1/Makefile.am 2021-04-01 09:01:29.000000000 +0000 +++ kio-recoll-1.31.0/Makefile.am 2021-04-23 08:51:05.000000000 +0000 @@ -58,7 +58,6 @@ librcl_LTLIBRARIES = librecoll.la librecoll_la_SOURCES = \ -aspell/aspell-local.h \ aspell/rclaspell.cpp \ aspell/rclaspell.h \ bincimapmime/convert.cc \ @@ -89,8 +88,6 @@ common/uproplist.h \ common/utf8fn.cpp \ common/utf8fn.h \ -index/webqueue.cpp \ -index/webqueue.h \ index/webqueuefetcher.cpp \ index/webqueuefetcher.h \ index/checkretryfailed.cpp \ @@ -101,8 +98,6 @@ index/fetcher.h \ index/fsfetcher.cpp \ index/fsfetcher.h \ -index/fsindexer.cpp \ -index/fsindexer.h \ index/idxdiags.h \ index/idxdiags.cpp \ index/idxstatus.h \ @@ -273,6 +268,8 @@ utils/wipedir.cpp \ utils/wipedir.h \ utils/workqueue.h \ +utils/x11mon.cpp \ +utils/x11mon.h \ utils/zlibut.cpp \ utils/zlibut.h \ xaposix/safefcntl.h \ @@ -290,7 +287,7 @@ # need it librecoll_la_LDFLAGS = -release $(VERSION) -no-undefined @NO_UNDEF_LINK_FLAG@ -librecoll_la_LIBADD = $(XSLT_LIBS) $(LIBXAPIAN) $(LIBICONV) $(LIBTHREADS) +librecoll_la_LIBADD = $(XSLT_LIBS) $(LIBXAPIAN) $(LIBICONV) $(X_LIBX11) $(LIBTHREADS) # There is probably a better way to do this. The KIO needs to be linked # with librecoll, but librecoll is installed into a non-standard place @@ -319,16 +316,19 @@ endif recollindex_SOURCES = \ - index/recollindex.cpp \ - index/checkindexed.cpp \ - index/checkindexed.h \ - index/indexer.cpp \ - index/indexer.h \ - index/rclmonprc.cpp \ - index/rclmonrcv.cpp \ - utils/x11mon.cpp \ - utils/x11mon.h -recollindex_LDADD = librecoll.la $(X_LIBX11) + index/checkindexed.cpp \ + index/checkindexed.h \ + index/fsindexer.cpp \ + index/fsindexer.h \ + index/indexer.cpp \ + index/indexer.h \ + index/rclmonprc.cpp \ + index/rclmonrcv.cpp \ + index/recollindex.cpp \ + index/webqueue.cpp \ + index/webqueue.h + +recollindex_LDADD = librecoll.la recollq_SOURCES = query/recollqmain.cpp recollq_LDADD = librecoll.la diff -Nru kio-recoll-1.30.1/Makefile.in kio-recoll-1.31.0/Makefile.in --- kio-recoll-1.30.1/Makefile.in 2021-04-12 13:01:46.000000000 +0000 +++ kio-recoll-1.31.0/Makefile.in 2021-04-27 08:13:46.000000000 +0000 @@ -147,7 +147,7 @@ am__DEPENDENCIES_1 = librecoll_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ - $(am__DEPENDENCIES_1) + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) am__dirstamp = $(am__leading_dot)dirstamp am_librecoll_la_OBJECTS = aspell/rclaspell.lo bincimapmime/convert.lo \ bincimapmime/mime-parsefull.lo \ @@ -156,38 +156,38 @@ common/webstore.lo common/cstr.lo common/rclconfig.lo \ common/rclinit.lo common/syngroups.lo common/textsplit.lo \ common/textsplitko.lo common/unacpp.lo common/utf8fn.lo \ - index/webqueue.lo index/webqueuefetcher.lo \ - index/checkretryfailed.lo index/exefetcher.lo index/fetcher.lo \ - index/fsfetcher.lo index/fsindexer.lo index/idxdiags.lo \ - index/idxstatus.lo index/mimetype.lo index/subtreelist.lo \ - internfile/extrameta.lo internfile/htmlparse.lo \ - internfile/internfile.lo internfile/mh_exec.lo \ - internfile/mh_execm.lo internfile/mh_html.lo \ - internfile/mh_mail.lo internfile/mh_mbox.lo \ - internfile/mh_text.lo internfile/mh_xslt.lo \ - internfile/mimehandler.lo internfile/myhtmlparse.lo \ - internfile/txtdcode.lo internfile/uncomp.lo query/docseq.lo \ - query/docseqdb.lo query/docseqhist.lo query/dynconf.lo \ - query/filtseq.lo query/plaintorich.lo query/qresultstore.lo \ - query/recollq.lo query/reslistpager.lo query/sortseq.lo \ - query/wasaparse.lo query/wasaparseaux.lo rcldb/daterange.lo \ - rcldb/expansiondbs.lo rcldb/rclabstract.lo \ - rcldb/rclabsfromtext.lo rcldb/rcldb.lo rcldb/rcldoc.lo \ - rcldb/rcldups.lo rcldb/rclquery.lo rcldb/rclterms.lo \ - rcldb/rclvalues.lo rcldb/searchdata.lo rcldb/searchdatatox.lo \ - rcldb/searchdataxml.lo rcldb/stemdb.lo rcldb/stoplist.lo \ - rcldb/synfamily.lo unac/unac.lo utils/appformime.lo \ - utils/base64.lo utils/cancelcheck.lo utils/chrono.lo \ - utils/circache.lo utils/closefrom.lo utils/cmdtalk.lo \ - utils/conftree.lo utils/copyfile.lo utils/cpuconf.lo \ - utils/dlib.lo utils/ecrontab.lo utils/execmd.lo \ - utils/fileudi.lo utils/fstreewalk.lo utils/hldata.lo \ - utils/idfile.lo utils/listmem.lo utils/log.lo utils/md5.lo \ - utils/md5ut.lo utils/mimeparse.lo utils/miniz.lo \ + index/webqueuefetcher.lo index/checkretryfailed.lo \ + index/exefetcher.lo index/fetcher.lo index/fsfetcher.lo \ + index/idxdiags.lo index/idxstatus.lo index/mimetype.lo \ + index/subtreelist.lo internfile/extrameta.lo \ + internfile/htmlparse.lo internfile/internfile.lo \ + internfile/mh_exec.lo internfile/mh_execm.lo \ + internfile/mh_html.lo internfile/mh_mail.lo \ + internfile/mh_mbox.lo internfile/mh_text.lo \ + internfile/mh_xslt.lo internfile/mimehandler.lo \ + internfile/myhtmlparse.lo internfile/txtdcode.lo \ + internfile/uncomp.lo query/docseq.lo query/docseqdb.lo \ + query/docseqhist.lo query/dynconf.lo query/filtseq.lo \ + query/plaintorich.lo query/qresultstore.lo query/recollq.lo \ + query/reslistpager.lo query/sortseq.lo query/wasaparse.lo \ + query/wasaparseaux.lo rcldb/daterange.lo rcldb/expansiondbs.lo \ + rcldb/rclabstract.lo rcldb/rclabsfromtext.lo rcldb/rcldb.lo \ + rcldb/rcldoc.lo rcldb/rcldups.lo rcldb/rclquery.lo \ + rcldb/rclterms.lo rcldb/rclvalues.lo rcldb/searchdata.lo \ + rcldb/searchdatatox.lo rcldb/searchdataxml.lo rcldb/stemdb.lo \ + rcldb/stoplist.lo rcldb/synfamily.lo unac/unac.lo \ + utils/appformime.lo utils/base64.lo utils/cancelcheck.lo \ + utils/chrono.lo utils/circache.lo utils/closefrom.lo \ + utils/cmdtalk.lo utils/conftree.lo utils/copyfile.lo \ + utils/cpuconf.lo utils/dlib.lo utils/ecrontab.lo \ + utils/execmd.lo utils/fileudi.lo utils/fstreewalk.lo \ + utils/hldata.lo utils/idfile.lo utils/listmem.lo utils/log.lo \ + utils/md5.lo utils/md5ut.lo utils/mimeparse.lo utils/miniz.lo \ utils/netcon.lo utils/pathut.lo utils/pxattr.lo \ utils/rclionice.lo utils/rclutil.lo utils/readfile.lo \ utils/smallut.lo utils/strmatcher.lo utils/transcode.lo \ - utils/utf8iter.lo utils/wipedir.lo utils/zlibut.lo + utils/utf8iter.lo utils/wipedir.lo utils/x11mon.lo \ + utils/zlibut.lo librecoll_la_OBJECTS = $(am_librecoll_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) @@ -196,12 +196,12 @@ librecoll_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(librecoll_la_LDFLAGS) $(LDFLAGS) -o $@ -am_recollindex_OBJECTS = index/recollindex.$(OBJEXT) \ - index/checkindexed.$(OBJEXT) index/indexer.$(OBJEXT) \ +am_recollindex_OBJECTS = index/checkindexed.$(OBJEXT) \ + index/fsindexer.$(OBJEXT) index/indexer.$(OBJEXT) \ index/rclmonprc.$(OBJEXT) index/rclmonrcv.$(OBJEXT) \ - utils/x11mon.$(OBJEXT) + index/recollindex.$(OBJEXT) index/webqueue.$(OBJEXT) recollindex_OBJECTS = $(am_recollindex_OBJECTS) -recollindex_DEPENDENCIES = librecoll.la $(am__DEPENDENCIES_1) +recollindex_DEPENDENCIES = librecoll.la am_recollq_OBJECTS = query/recollqmain.$(OBJEXT) recollq_OBJECTS = $(am_recollq_OBJECTS) recollq_DEPENDENCIES = librecoll.la @@ -237,12 +237,12 @@ index/$(DEPDIR)/checkindexed.Po \ index/$(DEPDIR)/checkretryfailed.Plo \ index/$(DEPDIR)/exefetcher.Plo index/$(DEPDIR)/fetcher.Plo \ - index/$(DEPDIR)/fsfetcher.Plo index/$(DEPDIR)/fsindexer.Plo \ + index/$(DEPDIR)/fsfetcher.Plo index/$(DEPDIR)/fsindexer.Po \ index/$(DEPDIR)/idxdiags.Plo index/$(DEPDIR)/idxstatus.Plo \ index/$(DEPDIR)/indexer.Po index/$(DEPDIR)/mimetype.Plo \ index/$(DEPDIR)/rclmonprc.Po index/$(DEPDIR)/rclmonrcv.Po \ index/$(DEPDIR)/recollindex.Po index/$(DEPDIR)/subtreelist.Plo \ - index/$(DEPDIR)/webqueue.Plo \ + index/$(DEPDIR)/webqueue.Po \ index/$(DEPDIR)/webqueuefetcher.Plo \ internfile/$(DEPDIR)/extrameta.Plo \ internfile/$(DEPDIR)/htmlparse.Plo \ @@ -292,7 +292,7 @@ utils/$(DEPDIR)/rclutil.Plo utils/$(DEPDIR)/readfile.Plo \ utils/$(DEPDIR)/smallut.Plo utils/$(DEPDIR)/strmatcher.Plo \ utils/$(DEPDIR)/transcode.Plo utils/$(DEPDIR)/utf8iter.Plo \ - utils/$(DEPDIR)/wipedir.Plo utils/$(DEPDIR)/x11mon.Po \ + utils/$(DEPDIR)/wipedir.Plo utils/$(DEPDIR)/x11mon.Plo \ utils/$(DEPDIR)/zlibut.Plo am__mv = mv -f CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ @@ -636,7 +636,6 @@ librcldir = $(libdir)/recoll librcl_LTLIBRARIES = librecoll.la librecoll_la_SOURCES = \ -aspell/aspell-local.h \ aspell/rclaspell.cpp \ aspell/rclaspell.h \ bincimapmime/convert.cc \ @@ -667,8 +666,6 @@ common/uproplist.h \ common/utf8fn.cpp \ common/utf8fn.h \ -index/webqueue.cpp \ -index/webqueue.h \ index/webqueuefetcher.cpp \ index/webqueuefetcher.h \ index/checkretryfailed.cpp \ @@ -679,8 +676,6 @@ index/fetcher.h \ index/fsfetcher.cpp \ index/fsfetcher.h \ -index/fsindexer.cpp \ -index/fsindexer.h \ index/idxdiags.h \ index/idxdiags.cpp \ index/idxstatus.h \ @@ -851,6 +846,8 @@ utils/wipedir.cpp \ utils/wipedir.h \ utils/workqueue.h \ +utils/x11mon.cpp \ +utils/x11mon.h \ utils/zlibut.cpp \ utils/zlibut.h \ xaposix/safefcntl.h \ @@ -867,19 +864,21 @@ # -version-info $(VERSION_INFO) would handle ABI compat issues, we don't # need it librecoll_la_LDFLAGS = -release $(VERSION) -no-undefined @NO_UNDEF_LINK_FLAG@ -librecoll_la_LIBADD = $(XSLT_LIBS) $(LIBXAPIAN) $(LIBICONV) $(LIBTHREADS) +librecoll_la_LIBADD = $(XSLT_LIBS) $(LIBXAPIAN) $(LIBICONV) $(X_LIBX11) $(LIBTHREADS) recollindex_SOURCES = \ - index/recollindex.cpp \ - index/checkindexed.cpp \ - index/checkindexed.h \ - index/indexer.cpp \ - index/indexer.h \ - index/rclmonprc.cpp \ - index/rclmonrcv.cpp \ - utils/x11mon.cpp \ - utils/x11mon.h + index/checkindexed.cpp \ + index/checkindexed.h \ + index/fsindexer.cpp \ + index/fsindexer.h \ + index/indexer.cpp \ + index/indexer.h \ + index/rclmonprc.cpp \ + index/rclmonrcv.cpp \ + index/recollindex.cpp \ + index/webqueue.cpp \ + index/webqueue.h -recollindex_LDADD = librecoll.la $(X_LIBX11) +recollindex_LDADD = librecoll.la recollq_SOURCES = query/recollqmain.cpp recollq_LDADD = librecoll.la xadump_SOURCES = query/xadump.cpp @@ -1429,8 +1428,6 @@ index/$(DEPDIR)/$(am__dirstamp): @$(MKDIR_P) index/$(DEPDIR) @: > index/$(DEPDIR)/$(am__dirstamp) -index/webqueue.lo: index/$(am__dirstamp) \ - index/$(DEPDIR)/$(am__dirstamp) index/webqueuefetcher.lo: index/$(am__dirstamp) \ index/$(DEPDIR)/$(am__dirstamp) index/checkretryfailed.lo: index/$(am__dirstamp) \ @@ -1441,8 +1438,6 @@ index/$(DEPDIR)/$(am__dirstamp) index/fsfetcher.lo: index/$(am__dirstamp) \ index/$(DEPDIR)/$(am__dirstamp) -index/fsindexer.lo: index/$(am__dirstamp) \ - index/$(DEPDIR)/$(am__dirstamp) index/idxdiags.lo: index/$(am__dirstamp) \ index/$(DEPDIR)/$(am__dirstamp) index/idxstatus.lo: index/$(am__dirstamp) \ @@ -1620,22 +1615,25 @@ utils/$(DEPDIR)/$(am__dirstamp) utils/wipedir.lo: utils/$(am__dirstamp) \ utils/$(DEPDIR)/$(am__dirstamp) +utils/x11mon.lo: utils/$(am__dirstamp) utils/$(DEPDIR)/$(am__dirstamp) utils/zlibut.lo: utils/$(am__dirstamp) utils/$(DEPDIR)/$(am__dirstamp) librecoll.la: $(librecoll_la_OBJECTS) $(librecoll_la_DEPENDENCIES) $(EXTRA_librecoll_la_DEPENDENCIES) $(AM_V_CXXLD)$(librecoll_la_LINK) -rpath $(librcldir) $(librecoll_la_OBJECTS) $(librecoll_la_LIBADD) $(LIBS) -index/recollindex.$(OBJEXT): index/$(am__dirstamp) \ - index/$(DEPDIR)/$(am__dirstamp) index/checkindexed.$(OBJEXT): index/$(am__dirstamp) \ index/$(DEPDIR)/$(am__dirstamp) +index/fsindexer.$(OBJEXT): index/$(am__dirstamp) \ + index/$(DEPDIR)/$(am__dirstamp) index/indexer.$(OBJEXT): index/$(am__dirstamp) \ index/$(DEPDIR)/$(am__dirstamp) index/rclmonprc.$(OBJEXT): index/$(am__dirstamp) \ index/$(DEPDIR)/$(am__dirstamp) index/rclmonrcv.$(OBJEXT): index/$(am__dirstamp) \ index/$(DEPDIR)/$(am__dirstamp) -utils/x11mon.$(OBJEXT): utils/$(am__dirstamp) \ - utils/$(DEPDIR)/$(am__dirstamp) +index/recollindex.$(OBJEXT): index/$(am__dirstamp) \ + index/$(DEPDIR)/$(am__dirstamp) +index/webqueue.$(OBJEXT): index/$(am__dirstamp) \ + index/$(DEPDIR)/$(am__dirstamp) recollindex$(EXEEXT): $(recollindex_OBJECTS) $(recollindex_DEPENDENCIES) $(EXTRA_recollindex_DEPENDENCIES) @rm -f recollindex$(EXEEXT) @@ -1697,7 +1695,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/exefetcher.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/fetcher.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/fsfetcher.Plo@am__quote@ # am--include-marker -@AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/fsindexer.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/fsindexer.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/idxdiags.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/idxstatus.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/indexer.Po@am__quote@ # am--include-marker @@ -1706,7 +1704,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/rclmonrcv.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/recollindex.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/subtreelist.Plo@am__quote@ # am--include-marker -@AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/webqueue.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/webqueue.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@index/$(DEPDIR)/webqueuefetcher.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@internfile/$(DEPDIR)/extrameta.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@internfile/$(DEPDIR)/htmlparse.Plo@am__quote@ # am--include-marker @@ -1787,7 +1785,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@utils/$(DEPDIR)/transcode.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@utils/$(DEPDIR)/utf8iter.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@utils/$(DEPDIR)/wipedir.Plo@am__quote@ # am--include-marker -@AMDEP_TRUE@@am__include@ @am__quote@utils/$(DEPDIR)/x11mon.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@utils/$(DEPDIR)/x11mon.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@utils/$(DEPDIR)/zlibut.Plo@am__quote@ # am--include-marker $(am__depfiles_remade): @@ -2372,8 +2370,8 @@ -rm -f query/wasaparse.cpp -rm -f query/wasaparse.hpp -test -z "$(BUILT_SOURCES)" || rm -f $(BUILT_SOURCES) -@MAKEPYTHONCHM_FALSE@@MAKEPYTHON_FALSE@@MAKEQT_FALSE@install-exec-local: @MAKEPYTHONCHM_FALSE@@MAKEPYTHON_FALSE@@MAKEQT_FALSE@clean-local: +@MAKEPYTHONCHM_FALSE@@MAKEPYTHON_FALSE@@MAKEQT_FALSE@install-exec-local: clean: clean-recursive clean-am: clean-binPROGRAMS clean-generic clean-librclLTLIBRARIES \ @@ -2401,7 +2399,7 @@ -rm -f index/$(DEPDIR)/exefetcher.Plo -rm -f index/$(DEPDIR)/fetcher.Plo -rm -f index/$(DEPDIR)/fsfetcher.Plo - -rm -f index/$(DEPDIR)/fsindexer.Plo + -rm -f index/$(DEPDIR)/fsindexer.Po -rm -f index/$(DEPDIR)/idxdiags.Plo -rm -f index/$(DEPDIR)/idxstatus.Plo -rm -f index/$(DEPDIR)/indexer.Po @@ -2410,7 +2408,7 @@ -rm -f index/$(DEPDIR)/rclmonrcv.Po -rm -f index/$(DEPDIR)/recollindex.Po -rm -f index/$(DEPDIR)/subtreelist.Plo - -rm -f index/$(DEPDIR)/webqueue.Plo + -rm -f index/$(DEPDIR)/webqueue.Po -rm -f index/$(DEPDIR)/webqueuefetcher.Plo -rm -f internfile/$(DEPDIR)/extrameta.Plo -rm -f internfile/$(DEPDIR)/htmlparse.Plo @@ -2491,7 +2489,7 @@ -rm -f utils/$(DEPDIR)/transcode.Plo -rm -f utils/$(DEPDIR)/utf8iter.Plo -rm -f utils/$(DEPDIR)/wipedir.Plo - -rm -f utils/$(DEPDIR)/x11mon.Po + -rm -f utils/$(DEPDIR)/x11mon.Plo -rm -f utils/$(DEPDIR)/zlibut.Plo -rm -f Makefile distclean-am: clean-am distclean-compile distclean-generic \ @@ -2562,7 +2560,7 @@ -rm -f index/$(DEPDIR)/exefetcher.Plo -rm -f index/$(DEPDIR)/fetcher.Plo -rm -f index/$(DEPDIR)/fsfetcher.Plo - -rm -f index/$(DEPDIR)/fsindexer.Plo + -rm -f index/$(DEPDIR)/fsindexer.Po -rm -f index/$(DEPDIR)/idxdiags.Plo -rm -f index/$(DEPDIR)/idxstatus.Plo -rm -f index/$(DEPDIR)/indexer.Po @@ -2571,7 +2569,7 @@ -rm -f index/$(DEPDIR)/rclmonrcv.Po -rm -f index/$(DEPDIR)/recollindex.Po -rm -f index/$(DEPDIR)/subtreelist.Plo - -rm -f index/$(DEPDIR)/webqueue.Plo + -rm -f index/$(DEPDIR)/webqueue.Po -rm -f index/$(DEPDIR)/webqueuefetcher.Plo -rm -f internfile/$(DEPDIR)/extrameta.Plo -rm -f internfile/$(DEPDIR)/htmlparse.Plo @@ -2652,7 +2650,7 @@ -rm -f utils/$(DEPDIR)/transcode.Plo -rm -f utils/$(DEPDIR)/utf8iter.Plo -rm -f utils/$(DEPDIR)/wipedir.Plo - -rm -f utils/$(DEPDIR)/x11mon.Po + -rm -f utils/$(DEPDIR)/x11mon.Plo -rm -f utils/$(DEPDIR)/zlibut.Plo -rm -f Makefile maintainer-clean-am: distclean-am maintainer-clean-generic diff -Nru kio-recoll-1.30.1/python/recoll/recoll/conftree.py kio-recoll-1.31.0/python/recoll/recoll/conftree.py --- kio-recoll-1.30.1/python/recoll/recoll/conftree.py 2021-02-26 08:43:17.000000000 +0000 +++ kio-recoll-1.31.0/python/recoll/recoll/conftree.py 2021-04-06 08:37:22.000000000 +0000 @@ -35,6 +35,7 @@ def __init__(self, confname, tildexp = False, readonly = True): self.submaps = {} + self.subkeys_unsorted = [] self.dotildexpand = tildexp self.readonly = readonly self.confname = confname @@ -81,6 +82,7 @@ else: submapkey = line #_debug("Submapkey: [%s]" % submapkey) + self.subkeys_unsorted.append(submapkey) continue nm, sep, value = line.partition(b'=') @@ -95,6 +97,9 @@ self.submaps[submapkey] = {} self.submaps[submapkey][nm] = value + def getSubKeys_unsorted(self): + return [k.decode('utf-8') for k in self.subkeys_unsorted] + def getbin(self, nm, sk = b''): '''Returns None if not found, empty string if found empty''' if type(nm) != type(b'') or type(sk) != type(b''): diff -Nru kio-recoll-1.30.1/qtgui/preview_plaintorich.cpp kio-recoll-1.31.0/qtgui/preview_plaintorich.cpp --- kio-recoll-1.30.1/qtgui/preview_plaintorich.cpp 2020-09-22 08:10:40.000000000 +0000 +++ kio-recoll-1.31.0/qtgui/preview_plaintorich.cpp 2021-04-14 07:12:38.000000000 +0000 @@ -89,11 +89,15 @@ // highlighting the spurious space. The space hack only work in a //
 section. Also: having  before the match
     // term causes the same problem (so not a possible fix).
-    string hackspace = m_spacehack? " " : "";
+    // Space does not seem to work any more (2021-04) ?
+    //   Zero Width Non Joiner works but is displayed as ? sometimes on windows.
+    //  nbsp seems to now work !
+    string hackspace = m_spacehack? " " : "";
     string startmarker{
-        "" + hackspace +
-            ""
-            };
+        "" +
+        hackspace +
+        "" 
+    };
     return startmarker;
 }
 
diff -Nru kio-recoll-1.30.1/qtgui/rclm_idx.cpp kio-recoll-1.31.0/qtgui/rclm_idx.cpp
--- kio-recoll-1.30.1/qtgui/rclm_idx.cpp	2021-04-01 09:08:03.000000000 +0000
+++ kio-recoll-1.31.0/qtgui/rclm_idx.cpp	2021-04-16 09:47:01.000000000 +0000
@@ -46,6 +46,7 @@
     switch (status.phase) {
     case DbIxStatus::DBIXS_NONE:phs=tr("None");break;
     case DbIxStatus::DBIXS_FILES: phs=tr("Updating");break;
+    case DbIxStatus::DBIXS_FLUSH: phs=tr("Flushing");break;
     case DbIxStatus::DBIXS_PURGE: phs=tr("Purge");break;
     case DbIxStatus::DBIXS_STEMDB: phs=tr("Stemdb");break;
     case DbIxStatus::DBIXS_CLOSING:phs=tr("Closing");break;
diff -Nru kio-recoll-1.30.1/qtgui/restable.cpp kio-recoll-1.31.0/qtgui/restable.cpp
--- kio-recoll-1.30.1/qtgui/restable.cpp	2021-04-10 13:22:45.000000000 +0000
+++ kio-recoll-1.31.0/qtgui/restable.cpp	2021-04-13 07:48:13.000000000 +0000
@@ -78,13 +78,13 @@
 static const char *settingskey_splittersizes="resTableSplitterSizes";
 
 //////////////////////////////////////////////////////////////////////////
-// Restable "pager". We use it to print details for a document in the 
+// Restable "pager". We use it to print details for a document in the
 // detail area
 ///
 class ResTablePager : public ResListPager {
 public:
     ResTablePager(ResTable *p)
-        : ResListPager(1, prefs.alwaysSnippets), m_parent(p) 
+        : ResListPager(1, prefs.alwaysSnippets), m_parent(p)
         {}
     virtual bool append(const string& data) override;
     virtual bool flush() override;
@@ -147,7 +147,7 @@
     if (m_table && m_table->m_model && m_table->m_detaildocnum >= 0) {
         int opts = m_table->m_ismainres ? ResultPopup::showExpand : 0;
         opts |= ResultPopup::showSaveOne;
-        QMenu *popup = ResultPopup::create(m_table, opts, 
+        QMenu *popup = ResultPopup::create(m_table, opts,
                                            m_table->m_model->getDocSource(),
                                            m_table->m_detaildoc);
         popup->popup(mapToGlobal(pos));
@@ -287,7 +287,7 @@
     }
 
     // Construct the actual list of column names
-    for (QStringList::const_iterator it = fields.begin(); 
+    for (QStringList::const_iterator it = fields.begin();
          it != fields.end(); it++) {
         m_fields.push_back((const char *)(it->toUtf8()));
         m_getters.push_back(chooseGetter(m_fields.back()));
@@ -367,7 +367,7 @@
     return (it == o_displayableFields.end()) ? u8s2qs(in) : it->second;
 }
 
-QVariant RecollModel::headerData(int idx, Qt::Orientation orientation, 
+QVariant RecollModel::headerData(int idx, Qt::Orientation orientation,
                                  int role) const
 {
     LOGDEB2("RecollModel::headerData: idx " << idx << " orientation " <<
@@ -375,7 +375,7 @@
             " role " << role << "\n");
     if (orientation == Qt::Vertical && role == Qt::DisplayRole) {
         if (idx < 26) {
-            return QString("%1/%2").arg(idx).arg(char('a'+idx)); 
+            return QString("%1/%2").arg(idx).arg(char('a'+idx));
         } else {
             return idx;
         }
@@ -402,11 +402,11 @@
             m_cachedfont = m_table->font();
             int fs = prefs.reslistfontsize <= fsadjusttable ?
                 prefs.reslistfontsize: prefs.reslistfontsize - fsadjusttable;
-            if (fs > 0) 
+            if (fs > 0)
                 m_cachedfont.setPointSize(fs);
         }
         return m_cachedfont;
-    }    
+    }
 
     if (!m_source || role != Qt::DisplayRole || !index.isValid() ||
         index.column() >= int(m_fields.size())) {
@@ -481,18 +481,18 @@
         return;
     }
     LOGDEB("RecollModel::sort(" << column << ", " << order << ")\n");
-    
+
     DocSeqSortSpec spec;
     if (column >= 0 && column < int(m_fields.size())) {
         spec.field = m_fields[column];
         if (!stringlowercmp("relevancyrating", spec.field) &&
             order != Qt::AscendingOrder) {
-            QMessageBox::warning(0, "Recoll", 
+            QMessageBox::warning(0, "Recoll",
                                  tr("Can't sort by inverse relevance"));
             QTimer::singleShot(0, m_table, SLOT(resetSort()));
             return;
         }
-        if (!stringlowercmp("date", spec.field) || 
+        if (!stringlowercmp("date", spec.field) ||
             !stringlowercmp("datetime", spec.field))
             spec.field = "mtime";
         spec.desc = order == Qt::AscendingOrder ? false : true;
@@ -500,7 +500,7 @@
     emit sortDataChanged(spec);
 }
 
-/////////////////////////// 
+///////////////////////////
 // ResTable panel methods
 
 // We use a custom delegate to display the cells because the base
@@ -514,7 +514,7 @@
     // need a modif to plaintorich to return the match count (easy),
     // and a way to pass an indicator from data(), a bit more
     // difficult. Anyway, the display seems fast enough as is.
-    void paint(QPainter *painter, const QStyleOptionViewItem &option, 
+    void paint(QPainter *painter, const QStyleOptionViewItem &option,
                const QModelIndex &index) const {
 
         QVariant value = index.data(Qt::DisplayRole);
@@ -651,12 +651,12 @@
 #endif
     setDefRowHeight();
 
-    connect(tableView->selectionModel(), 
+    connect(tableView->selectionModel(),
             SIGNAL(currentChanged(const QModelIndex&, const QModelIndex &)),
             this, SLOT(onTableView_currentChanged(const QModelIndex&)));
-    connect(tableView, SIGNAL(doubleClicked(const QModelIndex&)), 
+    connect(tableView, SIGNAL(doubleClicked(const QModelIndex&)),
             this, SLOT(onDoubleClick(const QModelIndex&)));
-    connect(tableView, SIGNAL(clicked(const QModelIndex&)), 
+    connect(tableView, SIGNAL(clicked(const QModelIndex&)),
             this, SLOT(onClicked(const QModelIndex&)));
 
     m_pager = new ResTablePager(this);
@@ -669,7 +669,7 @@
     m_detail->setOpenLinks(false);
     m_detail->init();
     // signals and slots connections
-    connect(m_detail, SIGNAL(anchorClicked(const QUrl &)), 
+    connect(m_detail, SIGNAL(anchorClicked(const QUrl &)),
             this, SLOT(linkWasClicked(const QUrl &)));
     splitter->addWidget(m_detail);
     splitter->setOrientation(Qt::Vertical);
@@ -753,7 +753,7 @@
     return false;
 }
 
-void ResTable::setRclMain(RclMain *m, bool ismain) 
+void ResTable::setRclMain(RclMain *m, bool ismain)
 {
     m_rclmain = m;
     m_ismainres = ismain;
@@ -774,13 +774,13 @@
     }
 
     new QShortcut(closeKeySeq, this, SLOT (close()));
-    connect(this, SIGNAL(previewRequested(Rcl::Doc)), 
+    connect(this, SIGNAL(previewRequested(Rcl::Doc)),
             m_rclmain, SLOT(startPreview(Rcl::Doc)));
-    connect(this, SIGNAL(editRequested(Rcl::Doc)), 
+    connect(this, SIGNAL(editRequested(Rcl::Doc)),
             m_rclmain, SLOT(startNativeViewer(Rcl::Doc)));
-    connect(this, SIGNAL(docSaveToFileClicked(Rcl::Doc)), 
+    connect(this, SIGNAL(docSaveToFileClicked(Rcl::Doc)),
             m_rclmain, SLOT(saveDocToFile(Rcl::Doc)));
-    connect(this, SIGNAL(showSnippets(Rcl::Doc)), 
+    connect(this, SIGNAL(showSnippets(Rcl::Doc)),
             m_rclmain, SLOT(showSnippets(Rcl::Doc)));
 }
 
@@ -833,9 +833,21 @@
 {
     LOGDEB1("setCurrentRowFromKbd: " << row << "\n");
     m_rowchangefromkbd = true;
-    tableView->setFocus(Qt::ShortcutFocusReason);                     
-    tableView->selectionModel()->setCurrentIndex(                     
-        m_model->index(row, 0),                                       
+    tableView->setFocus(Qt::ShortcutFocusReason);
+
+    // After calling setCurrentIndex(), currentChanged() gets called
+    // twice, once with row 0 and no selection, once with the actual
+    // target row and selection set. It uses this fact to discriminate
+    // this from hovering. For some reason, when row is zero, there is
+    // only one call. So, in this case, we first select row 1, and
+    // this so pretty hack gets things working
+    if (row == 0) {
+        tableView->selectionModel()->setCurrentIndex(
+            m_model->index(1, 0),
+            QItemSelectionModel::ClearAndSelect|QItemSelectionModel::Rows);
+    }
+    tableView->selectionModel()->setCurrentIndex(
+        m_model->index(row, 0),
         QItemSelectionModel::ClearAndSelect|QItemSelectionModel::Rows);
 }
 
@@ -979,7 +991,7 @@
     std::string tofile = qs2path(s);
     std::fstream fp;
     if (!path_streamopen(tofile, std::ios::out|std::ios::trunc,fp)) {
-        QMessageBox::warning(0, "Recoll", 
+        QMessageBox::warning(0, "Recoll",
                              tr("Can't open/create file: ") + s);
         return;
     }
@@ -1003,7 +1015,7 @@
     const vector fields = m_model->getFields();
     for (unsigned int i = 0; i < fields.size(); i++) {
         if (!spec.field.compare(m_model->baseField(fields[i]))) {
-            header->setSortIndicator(i, spec.desc ? 
+            header->setSortIndicator(i, spec.desc ?
                                      Qt::DescendingOrder : Qt::AscendingOrder);
             matched = true;
         }
@@ -1018,7 +1030,7 @@
     LOGDEB("ResTable::resetSort()\n");
     QHeaderView *header = tableView->horizontalHeader();
     if (header)
-        header->setSortIndicator(-1, Qt::AscendingOrder); 
+        header->setSortIndicator(-1, Qt::AscendingOrder);
     // the model's sort slot is not called by qt in this case (qt 4.7)
     if (m_model)
         m_model->sort(-1, Qt::AscendingOrder);
@@ -1078,8 +1090,8 @@
     }
     break;
 
-    case 'P': 
-    case 'E': 
+    case 'P':
+    case 'E':
     {
         if (what == 'P') {
             if (m_ismainres) {
@@ -1111,9 +1123,9 @@
     }
     break;
 
-    default: 
+    default:
         LOGERR("ResTable::linkWasClicked: bad link [" << ascurl << "]\n");
-        break;// ?? 
+        break;// ??
     }
 }
 
@@ -1127,7 +1139,7 @@
         onTableView_currentChanged(index);
     }
 }
-    
+
 void ResTable::onDoubleClick(const QModelIndex& index)
 {
     m_rowchangefromkbd = false;
@@ -1138,11 +1150,11 @@
         if (m_detaildocnum != index.row()) {
             m_detail->init();
             m_detaildocnum = index.row();
-            m_pager->displayDoc(theconfig, index.row(), m_detaildoc, 
+            m_pager->displayDoc(theconfig, index.row(), m_detaildoc,
                                 m_model->m_hdata);
         }
         m_detaildoc = doc;
-        if (m_detaildocnum >= 0) 
+        if (m_detaildocnum >= 0)
             emit editRequested(m_detaildoc);
     } else {
         m_detaildocnum = -1;
@@ -1161,7 +1173,7 @@
         if (selsz == 1) {
             opts |= ResultPopup::showSaveOne;
         } else if (selsz > 1 && !m_ismainres) {
-            // We don't show save multiple for the main list because not all 
+            // We don't show save multiple for the main list because not all
             // docs are necessary subdocs and multisave only works with those.
             opts |= ResultPopup::showSaveSel;
         }
@@ -1184,7 +1196,7 @@
 
 void ResTable::menuSaveToFile()
 {
-    if (m_detaildocnum >= 0) 
+    if (m_detaildocnum >= 0)
         emit docSaveToFileClicked(m_detaildoc);
 }
 
@@ -1211,9 +1223,9 @@
 
 void ResTable::menuPreviewParent()
 {
-    if (m_detaildocnum >= 0 && m_model &&  
+    if (m_detaildocnum >= 0 && m_model &&
         m_model->getDocSource()) {
-        Rcl::Doc pdoc = ResultPopup::getParent(m_model->getDocSource(), 
+        Rcl::Doc pdoc = ResultPopup::getParent(m_model->getDocSource(),
                                                m_detaildoc);
         if (pdoc.mimetype == "inode/directory") {
             emit editRequested(pdoc);
@@ -1246,7 +1258,7 @@
 
 void ResTable::menuEdit()
 {
-    if (m_detaildocnum >= 0) 
+    if (m_detaildocnum >= 0)
         emit editRequested(m_detaildoc);
 }
 void ResTable::menuEditAndQuit()
@@ -1261,25 +1273,25 @@
     if (act == 0)
         return;
     string cmd = qs2utf8s(act->data().toString());
-    if (m_detaildocnum >= 0) 
+    if (m_detaildocnum >= 0)
         emit openWithRequested(m_detaildoc, cmd);
 }
 
 void ResTable::menuCopyFN()
 {
-    if (m_detaildocnum >= 0) 
+    if (m_detaildocnum >= 0)
         ResultPopup::copyFN(m_detaildoc);
 }
 
 void ResTable::menuCopyPath()
 {
-    if (m_detaildocnum >= 0) 
+    if (m_detaildocnum >= 0)
         ResultPopup::copyPath(m_detaildoc);
 }
 
 void ResTable::menuCopyURL()
 {
-    if (m_detaildocnum >= 0) 
+    if (m_detaildocnum >= 0)
         ResultPopup::copyURL(m_detaildoc);
 }
 
@@ -1295,7 +1307,7 @@
             // it appears that the mouse event cancels it and it's not
             // shown). So let's do status bar if visible else tooltip.
             //  Menu trigger with no status bar -> no feedback...
-        
+
             // rclmain->showTrayMessage(msg);
             if (m_rclmain->statusBar()->isVisible()) {
                 m_rclmain->statusBar()->showMessage(msg, 1000);
@@ -1307,13 +1319,13 @@
                 QTimer::singleShot(1500, m_rclmain, SLOT(hideToolTip()));
             }
         }
-        
+
     }
 }
 
 void ResTable::menuExpand()
 {
-    if (m_detaildocnum >= 0) 
+    if (m_detaildocnum >= 0)
         emit docExpand(m_detaildoc);
 }
 
diff -Nru kio-recoll-1.30.1/qtgui/scbase.cpp kio-recoll-1.31.0/qtgui/scbase.cpp
--- kio-recoll-1.30.1/qtgui/scbase.cpp	2021-03-09 09:06:13.000000000 +0000
+++ kio-recoll-1.31.0/qtgui/scbase.cpp	2021-04-27 08:10:27.000000000 +0000
@@ -23,6 +23,7 @@
 #include 
 
 #include 
+#define LOGGER_LOCAL_LOGINC 2
 
 
 #include "recoll.h"
@@ -58,7 +59,7 @@
         auto ssc = qs2utf8s(sl.at(i));
         std::vector co_des_val;
         stringToStrings(ssc, co_des_val);
-        if (co_des_val.size() != 3) {
+        if (co_des_val.size() != 4) {
             LOGERR("Bad shortcut def in prefs: [" << ssc << "]\n");
             continue;
         }
@@ -68,8 +69,7 @@
         QString val = u8s2qs(co_des_val[3]);
         auto it = m->scvalues.find(id);
         if (it == m->scvalues.end()) {
-            m->scvalues[id] =
-                SCDef{id, ctxt, desc, QKeySequence(val), QKeySequence()};
+            m->scvalues[id] = SCDef{id, ctxt, desc, QKeySequence(val), QKeySequence()};
         } else {
             it->second.val = QKeySequence(val);
         }
diff -Nru kio-recoll-1.30.1/qtgui/uiprefs.ui kio-recoll-1.31.0/qtgui/uiprefs.ui
--- kio-recoll-1.30.1/qtgui/uiprefs.ui	2021-04-10 12:37:55.000000000 +0000
+++ kio-recoll-1.31.0/qtgui/uiprefs.ui	2021-04-14 07:12:38.000000000 +0000
@@ -7,7 +7,7 @@
     0
     0
     731
-    762
+    652
    
   
   
@@ -1532,8 +1532,11 @@
          
           
            
-            Work around QTBUG-78923 by inserting space before anchor text
+            Work around Tamil QTBUG-78923 by inserting space before anchor text
            
+             
+              The bug causes a strange circle characters to be displayed inside highlighted Tamil words. The workaround inserts an additional space character which appears to fix the problem.
+             
           
          
          
diff -Nru kio-recoll-1.30.1/query/docseq.cpp kio-recoll-1.31.0/query/docseq.cpp
--- kio-recoll-1.30.1/query/docseq.cpp	2020-09-05 07:43:16.000000000 +0000
+++ kio-recoll-1.31.0/query/docseq.cpp	2021-04-22 16:19:46.000000000 +0000
@@ -81,8 +81,7 @@
         }
     } else {
         if (m_fspec.isNotNull()) {
-            m_seq = 
-                std::shared_ptr(new DocSeqFiltered(m_config, m_seq, m_fspec));
+            m_seq = std::shared_ptr(new DocSeqFiltered(m_config, m_seq, m_fspec));
         } 
     }
     
diff -Nru kio-recoll-1.30.1/query/docseqdb.cpp kio-recoll-1.31.0/query/docseqdb.cpp
--- kio-recoll-1.30.1/query/docseqdb.cpp	2020-12-14 10:52:59.000000000 +0000
+++ kio-recoll-1.31.0/query/docseqdb.cpp	2021-04-22 16:26:43.000000000 +0000
@@ -31,14 +31,7 @@
 DocSequenceDb::DocSequenceDb(std::shared_ptr db,
                              std::shared_ptr q, const string &t, 
                              std::shared_ptr sdata) 
-    : DocSequence(t), m_db(db), m_q(q), m_sdata(sdata), m_fsdata(sdata),
-      m_rescnt(-1), 
-      m_queryBuildAbstract(true),
-      m_queryReplaceAbstract(false),
-      m_isFiltered(false),
-      m_isSorted(false),
-      m_needSetQuery(false),
-      m_lastSQStatus(true)
+    : DocSequence(t), m_db(db), m_q(q), m_sdata(sdata), m_fsdata(sdata)
 {
 }
 
@@ -161,8 +154,7 @@
     std::unique_lock locker(o_dblock);
     if (fs.isNotNull()) {
         // We build a search spec by adding a filtering layer to the base one.
-        m_fsdata = std::make_shared(
-            Rcl::SCLT_AND, m_sdata->getStemLang());
+        m_fsdata = std::make_shared(Rcl::SCLT_AND, m_sdata->getStemLang());
         Rcl::SearchDataClauseSub *cl = new Rcl::SearchDataClauseSub(m_sdata);
         m_fsdata->addClause(cl);
     
@@ -171,22 +163,19 @@
             case DocSeqFiltSpec::DSFS_MIMETYPE:
                 m_fsdata->addFiletype(fs.values[i]);
                 break;
-            case DocSeqFiltSpec::DSFS_QLANG:
-            {
+            case DocSeqFiltSpec::DSFS_QLANG: {
                 if (!m_q)
                     break;
                     
                 string reason;
-                auto sd = wasaStringToRcl(m_q->whatDb()->getConf(),
-                                          m_sdata->getStemLang(),
-                                          fs.values[i], reason);
+                auto sd = wasaStringToRcl(
+                    m_q->whatDb()->getConf(), m_sdata->getStemLang(), fs.values[i], reason);
                 if (sd)  {
-                    Rcl::SearchDataClauseSub *cl1 = 
-                        new Rcl::SearchDataClauseSub(sd);
+                    Rcl::SearchDataClauseSub *cl1 = new Rcl::SearchDataClauseSub(sd);
                     m_fsdata->addClause(cl1);
                 }
             }
-            break;
+                break;
             default:
                 break;
             }
@@ -226,8 +215,7 @@
     m_lastSQStatus = m_q->setQuery(m_fsdata);
     if (!m_lastSQStatus) {
         m_reason = m_q->getReason();
-        LOGERR("DocSequenceDb::setQuery: rclquery::setQuery failed: " <<
-               m_reason << "\n");
+        LOGERR("DocSequenceDb::setQuery: rclquery::setQuery failed: " << m_reason << "\n");
     }
     return m_lastSQStatus;
 }
@@ -241,4 +229,3 @@
         return false;
     }
 }
-
diff -Nru kio-recoll-1.30.1/query/docseqdb.h kio-recoll-1.31.0/query/docseqdb.h
--- kio-recoll-1.30.1/query/docseqdb.h	2020-09-05 07:43:16.000000000 +0000
+++ kio-recoll-1.31.0/query/docseqdb.h	2021-04-22 16:27:38.000000000 +0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 2004 J.F.Dockes
+/* Copyright (C) 2004-2021 J.F.Dockes
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
  *   the Free Software Foundation; either version 2 of the License, or
@@ -25,10 +25,10 @@
 
 /** A DocSequence from a Db query */
 class DocSequenceDb : public DocSequence {
- public:
+public:
     DocSequenceDb(std::shared_ptr db,
                   std::shared_ptr q, const std::string &t, 
-          std::shared_ptr sdata);
+                  std::shared_ptr sdata);
     virtual ~DocSequenceDb() {}
     virtual bool getDoc(int num, Rcl::Doc &doc, std::string * = 0) override;
     virtual int getResCnt() override;
@@ -55,7 +55,7 @@
     }
 
     virtual bool snippetsCapable() override {
-    return true;
+        return true;
     }
     virtual std::string title() override;
 
@@ -68,13 +68,13 @@
     std::shared_ptr      m_q;
     std::shared_ptr m_sdata;
     std::shared_ptr m_fsdata; // Filtered 
-    int                      m_rescnt;
-    bool                     m_queryBuildAbstract;
-    bool                     m_queryReplaceAbstract;
-    bool                     m_isFiltered;
-    bool                     m_isSorted;
-    bool   m_needSetQuery; // search data changed, need to reapply before fetch
-    bool   m_lastSQStatus;
+    int  m_rescnt{-1};
+    bool m_queryBuildAbstract{true};
+    bool m_queryReplaceAbstract{false};
+    bool m_isFiltered{false};
+    bool m_isSorted{false};
+    bool m_needSetQuery{false}; // search data changed, need to reapply before fetch
+    bool m_lastSQStatus{true};
     bool setQuery();
 };
 
diff -Nru kio-recoll-1.30.1/query/docseqhist.cpp kio-recoll-1.31.0/query/docseqhist.cpp
--- kio-recoll-1.30.1/query/docseqhist.cpp	2020-09-05 07:43:16.000000000 +0000
+++ kio-recoll-1.31.0/query/docseqhist.cpp	2021-04-22 16:29:23.000000000 +0000
@@ -125,31 +125,31 @@
 {
     // Retrieve history list
     if (!m_hist)
-    return false;
+        return false;
     if (m_history.empty())
-    m_history = getDocHistory(m_hist);
+        m_history = getDocHistory(m_hist);
 
     if (num < 0 || num >= (int)m_history.size())
-    return false;
+        return false;
 
     // We get the history oldest first, but our users expect newest first
     RclDHistoryEntry& hentry = m_history[m_history.size() - 1 - num];
 
     if (sh) {
-    if (m_prevtime < 0 || abs(m_prevtime - hentry.unixtime) > 86400) {
-        m_prevtime = hentry.unixtime;
-        time_t t = (time_t)(hentry.unixtime);
-        *sh = string(ctime(&t));
-        // Get rid of the final \n in ctime
-        sh->erase(sh->length()-1);
-    } else {
-        sh->erase();
+        if (m_prevtime < 0 || abs(m_prevtime - hentry.unixtime) > 86400) {
+            m_prevtime = hentry.unixtime;
+            time_t t = (time_t)(hentry.unixtime);
+            *sh = string(ctime(&t));
+            // Get rid of the final \n in ctime
+            sh->erase(sh->length()-1);
+        } else {
+            sh->erase();
         }
     }
 
     bool ret = m_db->getDoc(hentry.udi, hentry.dbdir, doc);
     if (!ret || doc.pc == -1) {
-    doc.url = "UNKNOWN";
+        doc.url = "UNKNOWN";
         doc.ipath = "";
     }
 
@@ -163,7 +163,7 @@
 int DocSequenceHistory::getResCnt()
 {    
     if (m_history.empty())
-    m_history = getDocHistory(m_hist);
+        m_history = getDocHistory(m_hist);
     return int(m_history.size());
 }
 
diff -Nru kio-recoll-1.30.1/query/filtseq.cpp kio-recoll-1.31.0/query/filtseq.cpp
--- kio-recoll-1.30.1/query/filtseq.cpp	2020-09-06 13:59:19.000000000 +0000
+++ kio-recoll-1.31.0/query/filtseq.cpp	2021-04-22 16:29:52.000000000 +0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005 J.F.Dockes
+/* Copyright (C) 2005-2021 J.F.Dockes
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
  *   the Free Software Foundation; either version 2 of the License, or
@@ -24,69 +24,64 @@
 
 static bool filter(const DocSeqFiltSpec& fs, const Rcl::Doc *x)
 {
-    LOGDEB2("  Filter: ncrits "  << (fs.crits.size()) << "\n" );
-    // Compare using each criterion in term. We're doing an or:
-    // 1st ok ends 
+    LOGDEB2("  Filter: ncrits " << fs.crits.size() << "\n");
+    // Compare using each criterion in term. We're doing an or: 1st ok ends 
     for (unsigned int i = 0; i < fs.crits.size(); i++) {
-    switch (fs.crits[i]) {
-    case DocSeqFiltSpec::DSFS_MIMETYPE:
-        LOGDEB2(" filter: MIMETYPE: me ["  << (fs.values[i]) << "] doc ["  << (x->mimetype) << "]\n" );
-        if (x->mimetype == fs.values[i])
-        return true;
-        break;
-    case DocSeqFiltSpec::DSFS_QLANG:
-    {
-        LOGDEB(" filter: QLANG ["  << (fs.values[i]) << "]!!\n" );
-    }
-    break;
-    case DocSeqFiltSpec::DSFS_PASSALL:
-        return true;
-    }
+        switch (fs.crits[i]) {
+        case DocSeqFiltSpec::DSFS_MIMETYPE:
+            LOGDEB2(" filter: MIMETYPE: me [" << fs.values[i] << "] doc [" << x->mimetype << "]\n");
+            if (x->mimetype == fs.values[i])
+                return true;
+            break;
+        case DocSeqFiltSpec::DSFS_QLANG: {
+            LOGDEB(" filter: QLANG [" << fs.values[i] << "]!!\n");
+        }
+            break;
+        case DocSeqFiltSpec::DSFS_PASSALL:
+            return true;
+        }
     }
     // Did all comparisons
     return false;
 } 
 
-DocSeqFiltered::DocSeqFiltered(RclConfig *conf, std::shared_ptr iseq, 
-                   DocSeqFiltSpec &filtspec)
-    :  DocSeqModifier(iseq), m_config(conf)
+DocSeqFiltered::DocSeqFiltered(
+    RclConfig *conf, std::shared_ptr iseq, DocSeqFiltSpec &filtspec)
+    : DocSeqModifier(iseq), m_config(conf)
 {
     setFiltSpec(filtspec);
 }
 
 bool DocSeqFiltered::setFiltSpec(const DocSeqFiltSpec &filtspec)
 {
-    LOGDEB0("DocSeqFiltered::setFiltSpec\n" );
+    LOGDEB0("DocSeqFiltered::setFiltSpec\n");
     for (unsigned int i = 0; i < filtspec.crits.size(); i++) {
-    switch (filtspec.crits[i]) {
-    case DocSeqFiltSpec::DSFS_MIMETYPE:
-        m_spec.orCrit(filtspec.crits[i], filtspec.values[i]);
-        break;
-    case DocSeqFiltSpec::DSFS_QLANG:
-    {
-        // There are very few lang constructs that we can
-        // interpret. The default config uses rclcat:value
-        // only. That will be all for now...
-        string val = filtspec.values[i];
-        if (val.find("rclcat:") == 0) {
-        string catg = val.substr(7);
-        vector tps;
-        m_config->getMimeCatTypes(catg, tps);
-        for (vector::const_iterator it = tps.begin();
-             it != tps.end(); it++) {
-            LOGDEB2("Adding mime: [" << *it << "]\n");
-            m_spec.orCrit(DocSeqFiltSpec::DSFS_MIMETYPE, *it);
+        switch (filtspec.crits[i]) {
+        case DocSeqFiltSpec::DSFS_MIMETYPE:
+            m_spec.orCrit(filtspec.crits[i], filtspec.values[i]);
+            break;
+        case DocSeqFiltSpec::DSFS_QLANG: {
+            // There are very few lang constructs that we can interpret. The
+            // default config uses rclcat:value only. That will be all for now...
+            string val = filtspec.values[i];
+            if (val.find("rclcat:") == 0) {
+                string catg = val.substr(7);
+                vector tps;
+                m_config->getMimeCatTypes(catg, tps);
+                for (const auto& mime : tps) {
+                    LOGDEB2("Adding mime: [" << mime << "]\n");
+                    m_spec.orCrit(DocSeqFiltSpec::DSFS_MIMETYPE, mime);
+                }
+            }
         }
+            break;
+        default:
+            break;
         }
     }
-    break;
-    default:
-        break;
-    }
-    }
     // If m_spec ends up empty, pass everything, better than filtering all.
     if (m_spec.crits.empty()) {
-    m_spec.orCrit(DocSeqFiltSpec::DSFS_PASSALL, "");
+        m_spec.orCrit(DocSeqFiltSpec::DSFS_PASSALL, "");
     }
     m_dbindices.clear();
     return true;
@@ -94,31 +89,31 @@
 
 bool DocSeqFiltered::getDoc(int idx, Rcl::Doc &doc, string *)
 {
-    LOGDEB2("DocSeqFiltered::getDoc() fetching "  << (idx) << "\n" );
+    LOGDEB2("DocSeqFiltered::getDoc() fetching " << idx << "\n");
 
     if (idx >= (int)m_dbindices.size()) {
-    // Have to fetch docs and filter until we get enough or
-    // fail
-    m_dbindices.reserve(idx+1);
-
-    // First backend seq doc we fetch is the one after last stored 
-    int backend_idx = m_dbindices.size() > 0 ? m_dbindices.back() + 1 : 0;
-
-    // Loop until we get enough docs
-    Rcl::Doc tdoc;
-    while (idx >= (int)m_dbindices.size()) {
-        if (!m_seq->getDoc(backend_idx, tdoc)) 
-        return false;
-        if (filter(m_spec, &tdoc)) {
-        m_dbindices.push_back(backend_idx);
+        // Have to fetch docs and filter until we get enough or
+        // fail
+        m_dbindices.reserve(idx+1);
+
+        // First backend seq doc we fetch is the one after last stored 
+        int backend_idx = m_dbindices.size() > 0 ? m_dbindices.back() + 1 : 0;
+
+        // Loop until we get enough docs
+        Rcl::Doc tdoc;
+        while (idx >= (int)m_dbindices.size()) {
+            if (!m_seq->getDoc(backend_idx, tdoc)) 
+                return false;
+            if (filter(m_spec, &tdoc)) {
+                m_dbindices.push_back(backend_idx);
+            }
+            backend_idx++;
         }
-        backend_idx++;
-    }
-    doc = tdoc;
+        doc = tdoc;
     } else {
-    // The corresponding backend indice is already known
-    if (!m_seq->getDoc(m_dbindices[idx], doc)) 
-        return false;
+        // The corresponding backend indice is already known
+        if (!m_seq->getDoc(m_dbindices[idx], doc)) 
+            return false;
     }
     return true;
 }
diff -Nru kio-recoll-1.30.1/query/filtseq.h kio-recoll-1.31.0/query/filtseq.h
--- kio-recoll-1.30.1/query/filtseq.h	2020-09-05 07:43:16.000000000 +0000
+++ kio-recoll-1.31.0/query/filtseq.h	2021-04-22 16:28:05.000000000 +0000
@@ -29,20 +29,22 @@
 /** 
  * A filtered sequence is created from another one by selecting entries
  * according to the given criteria.
+ * Note that this class can only filter on mime type (or rclcatg) at the moment,
+ * and is only used for history. Normal query filtering is performed by adding a
+ * clause to the Xapian query.
  */
 class DocSeqFiltered : public DocSeqModifier {
 public:
-    DocSeqFiltered(RclConfig *conf, std::shared_ptr iseq, 
-           DocSeqFiltSpec &filtspec);
+    DocSeqFiltered(RclConfig *conf, std::shared_ptr iseq, DocSeqFiltSpec &filtspec);
     virtual ~DocSeqFiltered() {}
     virtual bool canFilter() {return true;}
     virtual bool setFiltSpec(const DocSeqFiltSpec &filtspec);
     virtual bool getDoc(int num, Rcl::Doc &doc, std::string *sh = 0);
     virtual int getResCnt() {return m_seq->getResCnt();}
- private:
+private:
     RclConfig     *m_config;    
     DocSeqFiltSpec m_spec;
-    std::vector    m_dbindices;
+    std::vector m_dbindices;
 };
 
 #endif /* _FILTSEQ_H_INCLUDED_ */
diff -Nru kio-recoll-1.30.1/query/recollq.cpp kio-recoll-1.31.0/query/recollq.cpp
--- kio-recoll-1.30.1/query/recollq.cpp	2021-01-05 18:54:16.000000000 +0000
+++ kio-recoll-1.31.0/query/recollq.cpp	2021-04-21 10:51:55.000000000 +0000
@@ -66,7 +66,7 @@
         std::vector snippets;
         std::ostringstream str;
         if (query.makeDocAbstract(doc, snippets, snipcount, -1, true)) {
-            for (const auto snippet : snippets) {
+            for (const auto& snippet : snippets) {
                 str << snippet.page << " : " << snippet.snippet << endl;
             }
         }
@@ -467,7 +467,7 @@
                 << doc.fbytes << "\tbytes" << "\t"
                 <<  endl;
             if (op_flags & OPT_m) {
-                for (const auto ent : doc.meta) {
+                for (const auto& ent : doc.meta) {
                     cout << ent.first << " = " << ent.second << endl;
                 }
             }
diff -Nru kio-recoll-1.30.1/query/sortseq.cpp kio-recoll-1.31.0/query/sortseq.cpp
--- kio-recoll-1.30.1/query/sortseq.cpp	2020-09-05 07:43:16.000000000 +0000
+++ kio-recoll-1.31.0/query/sortseq.cpp	2021-04-22 06:44:04.000000000 +0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005 J.F.Dockes
+/* Copyright (C) 2005-2021 J.F.Dockes
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
  *   the Free Software Foundation; either version 2 of the License, or
@@ -16,10 +16,9 @@
  */
 #include "autoconfig.h"
 
-#include "sortseq.h"
-
 #include 
 
+#include "sortseq.h"
 #include "log.h"
 
 using std::string;
@@ -31,15 +30,14 @@
 
     // It's not too clear in the std::sort doc what this should do. This 
     // behaves as operator< 
-    int operator()(const Rcl::Doc *x, const Rcl::Doc *y) 
-    { 
-    LOGDEB1("Comparing .. \n" );
-
-    const auto xit = x->meta.find(ss.field);
-    const auto yit = y->meta.find(ss.field);
-    if (xit == x->meta.end() || yit == y->meta.end())
-        return 0;
-    return ss.desc ? yit->second < xit->second : xit->second < yit->second;
+    int operator()(const Rcl::Doc *x, const Rcl::Doc *y) { 
+        LOGDEB1("Comparing .. \n" );
+
+        const auto xit = x->meta.find(ss.field);
+        const auto yit = y->meta.find(ss.field);
+        if (xit == x->meta.end() || yit == y->meta.end())
+            return 0;
+        return ss.desc ? yit->second < xit->second : xit->second < yit->second;
     } 
 };
 
@@ -52,16 +50,16 @@
     m_docs.resize(count);
     int i;
     for (i = 0; i < count; i++) {
-    if (!m_seq->getDoc(i, m_docs[i])) {
-        LOGERR("DocSeqSorted: getDoc failed for doc "  << (i) << "\n" );
-        count = i;
-        break;
-    }
+        if (!m_seq->getDoc(i, m_docs[i])) {
+            LOGERR("DocSeqSorted: getDoc failed for doc " << i << "\n");
+            count = i;
+            break;
+        }
     }
     m_docs.resize(count);
     m_docsp.resize(count);
     for (i = 0; i < count; i++)
-    m_docsp[i] = &m_docs[i];
+        m_docsp[i] = &m_docs[i];
 
     CompareDocs cmp(sortspec);
     sort(m_docsp.begin(), m_docsp.end(), cmp);
@@ -70,10 +68,9 @@
 
 bool DocSeqSorted::getDoc(int num, Rcl::Doc &doc, string *)
 {
-    LOGDEB("DocSeqSorted::getDoc("  << (num) << ")\n" );
+    LOGDEB("DocSeqSorted::getDoc(" << num << ")\n");
     if (num < 0 || num >= int(m_docsp.size()))
-    return false;
+        return false;
     doc = *m_docsp[num];
     return true;
 }
-
diff -Nru kio-recoll-1.30.1/query/wasaparseaux.cpp kio-recoll-1.31.0/query/wasaparseaux.cpp
--- kio-recoll-1.30.1/query/wasaparseaux.cpp	2020-12-14 10:52:59.000000000 +0000
+++ kio-recoll-1.31.0/query/wasaparseaux.cpp	2021-04-24 11:31:47.000000000 +0000
@@ -50,19 +50,6 @@
     return sd;
 }
 
-WasaParserDriver::WasaParserDriver(const RclConfig *c, const std::string sl, 
-                                   const std::string& as)
-    : m_stemlang(sl), m_autosuffs(as), m_config(c),
-      m_index(0), m_result(0), m_haveDates(false), 
-      m_maxSize(-1), m_minSize(-1)
-{
-
-}
-
-WasaParserDriver::~WasaParserDriver()
-{
-}
-
 SearchData *WasaParserDriver::parse(const std::string& in)
 {
     m_input = in;
@@ -83,13 +70,11 @@
         return m_result;
 
     // Set the top level filters (types, dates, size)
-    for (vector::const_iterator it = m_filetypes.begin();
-         it != m_filetypes.end(); it++) {
-        m_result->addFiletype(*it);
-    }
-    for (vector::const_iterator it = m_nfiletypes.begin();
-         it != m_nfiletypes.end(); it++) {
-        m_result->remFiletype(*it);
+    for (const auto& ft : m_filetypes) {
+        m_result->addFiletype(ft);
+    }
+    for (const auto& ft : m_nfiletypes) {
+        m_result->remFiletype(ft);
     }
     if (m_haveDates) {
         m_result->setDateSpan(&m_dates);
@@ -100,6 +85,10 @@
     if (m_maxSize != -1) {
         m_result->setMaxSize(m_maxSize);
     }
+    if (m_subSpec != Rcl::SearchData::SUBDOC_ANY) {
+        m_result->setSubSpec(m_subSpec);
+    }
+    
     //if (m_result)  m_result->dump(cout);
     return m_result;
 }
@@ -122,8 +111,7 @@
 
 // Add clause to query, handling special pseudo-clauses for size/date
 // etc. (mostly determined on field name).
-bool WasaParserDriver::addClause(SearchData *sd, 
-                                 SearchDataClauseSimple* cl)
+bool WasaParserDriver::addClause(SearchData *sd, SearchDataClauseSimple* cl)
 {
     if (cl->getfield().empty()) {
         // Simple clause with empty field spec.
@@ -132,7 +120,7 @@
         if (!m_autosuffs.empty()) {
             vector asfv;
             if (stringToStrings(m_autosuffs, asfv)) {
-                if (find_if(asfv.begin(), asfv.end(), 
+                if (find_if(asfv.begin(), asfv.end(),
                             StringIcmpPred(cl->gettext())) != asfv.end()) {
                     cl->setfield("ext");
                     cl->addModifier(SearchDataClause::SDCM_NOSTEMMING);
@@ -156,6 +144,13 @@
         return false;
     } 
 
+    // Filtering for standalone- or sub-documents
+    if (!fld.compare("issub")) {
+        m_subSpec = atoi(cl->gettext().c_str());
+        delete cl;
+        return false;
+    } 
+
     if (!fld.compare("rclcat") || !fld.compare("type")) {
         vector mtypes;
         if (m_config && m_config->getMimeCatTypes(cl->gettext(), mtypes)) {
@@ -231,8 +226,7 @@
 
     if (!fld.compare("dir")) {
         // dir filtering special case
-        SearchDataClausePath *nclause = 
-            new SearchDataClausePath(cl->gettext(), cl->getexclude());
+        SearchDataClausePath *nclause = new SearchDataClausePath(cl->gettext(), cl->getexclude());
         delete cl;
         return sd->addClause(nclause);
     }
@@ -258,8 +252,7 @@
         }
 
         if (tp != SCLT_FILENAME) {
-            SearchDataClauseSimple *ncl = 
-                new SearchDataClauseSimple(tp, ns, ofld);
+            SearchDataClauseSimple *ncl = new SearchDataClauseSimple(tp, ns, ofld);
             delete cl;
             return sd->addClause(ncl);
         }
diff -Nru kio-recoll-1.30.1/query/wasaparserdriver.h kio-recoll-1.31.0/query/wasaparserdriver.h
--- kio-recoll-1.30.1/query/wasaparserdriver.h	2020-12-13 08:07:13.000000000 +0000
+++ kio-recoll-1.31.0/query/wasaparserdriver.h	2021-04-24 06:18:31.000000000 +0000
@@ -22,14 +22,12 @@
 #include 
 
 #include "smallut.h"
+#include "searchdata.h"
 
 class WasaParserDriver;
-namespace Rcl {
-    class SearchData;
-    class SearchDataClauseSimple;
-}
+
 namespace yy {
-    class parser;
+class parser;
 }
 
 class RclConfig;
@@ -37,9 +35,10 @@
 class WasaParserDriver {
 public:
     
-    WasaParserDriver(const RclConfig *c, const std::string sl, 
-                     const std::string& as);
-    ~WasaParserDriver();
+    WasaParserDriver(const RclConfig *c, const std::string sl, const std::string& as)
+        : m_stemlang(sl), m_autosuffs(as), m_config(c) {}
+        
+    ~WasaParserDriver() {}
     
     Rcl::SearchData *parse(const std::string&);
     bool addClause(Rcl::SearchData *sd, Rcl::SearchDataClauseSimple* cl);
@@ -67,20 +66,20 @@
     // input string.
     std::string m_input;
     // Current position in m_input
-    unsigned int m_index;
+    unsigned int m_index{0};
     // Characters pushed-back, ready for next getchar.
     std::stack m_returns;
     // Result, set by parser.
-    Rcl::SearchData *m_result;
+    Rcl::SearchData *m_result{nullptr};
 
     // Storage for top level filters
     std::vector  m_filetypes; 
     std::vector  m_nfiletypes;
-    bool                      m_haveDates;
+    bool                      m_haveDates{false};
     DateInterval              m_dates; // Restrict to date interval
-    int64_t                   m_maxSize;
-    int64_t                   m_minSize;
-
+    int64_t                   m_maxSize{-1};
+    int64_t                   m_minSize{-1};
+    int                       m_subSpec{Rcl::SearchData::SUBDOC_ANY};
     std::string m_reason;
 
     // Let the quoted string reader store qualifiers in there, simpler
diff -Nru kio-recoll-1.30.1/rcldb/rclabsfromtext.cpp kio-recoll-1.31.0/rcldb/rclabsfromtext.cpp
--- kio-recoll-1.30.1/rcldb/rclabsfromtext.cpp	2021-04-01 16:22:26.000000000 +0000
+++ kio-recoll-1.31.0/rcldb/rclabsfromtext.cpp	2021-04-13 08:42:00.000000000 +0000
@@ -36,7 +36,7 @@
 
 using namespace std;
 
-// #define DEBUGABSTRACT  
+#undef DEBUGABSTRACT  
 #ifdef DEBUGABSTRACT
 #define LOGABS LOGDEB
 #else
@@ -221,8 +221,7 @@
                 // Term group (phrase/near) handling
                 m_plists[dumb].push_back(pos);
                 m_gpostobytes[pos] = pair(bts, bte);
-                LOGDEB1("Recorded bpos for pos " << pos << ": " << bts << " " <<
-                        bte << "\n");
+                LOGDEB1("Recorded bpos for pos " << pos << ": " << bts << " " << bte << "\n");
             }
         }
 #ifdef COMPUTE_HLZONES
@@ -278,6 +277,20 @@
     // After the text is split: use the group terms positions lists to
     // find the group matches.
     void updgroups() {
+        // Possibly store current incomplete fragment (if match was
+        // close to the end of the text, so we did not close it):
+        if (m_curtermcoef != 0.0) {
+            m_fragments.push_back(
+                MatchFragment(m_curfrag.first, m_curfrag.second, m_curfragcoef,
+#ifdef COMPUTE_HLZONES
+                              m_curhlzones,
+#endif
+                              m_curhitpos, m_curterm));
+                m_totalcoef += m_curfragcoef;
+                m_curfragcoef = 0.0;
+                m_curtermcoef = 0.0;
+        }
+
         LOGDEB("TextSplitABS: stored total " << m_fragments.size() <<
                " fragments" << endl);
         vector tboffs;
diff -Nru kio-recoll-1.30.1/rcldb/rcldb.cpp kio-recoll-1.31.0/rcldb/rcldb.cpp
--- kio-recoll-1.30.1/rcldb/rcldb.cpp	2021-04-01 16:22:26.000000000 +0000
+++ kio-recoll-1.31.0/rcldb/rcldb.cpp	2021-04-24 06:42:16.000000000 +0000
@@ -63,6 +63,7 @@
 #include "rclaspell.h"
 #endif
 #include "zlibut.h"
+#include "idxstatus.h"
 
 #ifndef XAPIAN_AT_LEAST
 // Added in Xapian 1.4.2. Define it here for older versions
@@ -1452,14 +1453,11 @@
             return false;
         }
 
-        list asuggs;
         string reason;
-        if (!m_aspell->suggest(*this, term, asuggs, reason)) {
-            LOGERR("Db::getSpellingSuggestions: aspell failed: " << reason <<
-                   "\n");
+        if (!m_aspell->suggest(*this, term, suggs, reason)) {
+            LOGERR("Db::getSpellingSuggestions: aspell failed: " << reason << "\n");
             return false;
         }
-        suggs = vector(asuggs.begin(), asuggs.end());
 #endif
     } else {
 #ifdef TESTING_XAPIAN_SPELL
@@ -1569,8 +1567,7 @@
             // There is no way in hell we could have an idea of the
             // charset here, so let's hope it's ascii or utf-8. We call
             // transcode to strip the bad chars and pray
-            if (transcode(path_getsimple(doc.ipath), utf8ipathlast,
-                          "UTF-8", "UTF-8")) {
+            if (transcode(path_getsimple(doc.ipath), utf8ipathlast, "UTF-8", "UTF-8")) {
                 splitter.text_to_words(utf8ipathlast);
             }
         }
@@ -1595,14 +1592,12 @@
             splitter.curpos = 0;
             newdocument.add_posting(wrap_prefix(pathelt_prefix),
                                     splitter.basepos + splitter.curpos++);
-            for (vector::iterator it = vpath.begin(); 
-                 it != vpath.end(); it++){
-                if (it->length() > 230) {
-                    // Just truncate it. May still be useful because
-                    // of wildcards
-                    *it = it->substr(0, 230);
+            for (auto& elt : vpath) {
+                if (elt.length() > 230) {
+                    // Just truncate it. May still be useful because of wildcards
+                    elt = elt.substr(0, 230);
                 }
-                newdocument.add_posting(wrap_prefix(pathelt_prefix) + *it, 
+                newdocument.add_posting(wrap_prefix(pathelt_prefix) + elt, 
                                         splitter.basepos + splitter.curpos++);
             }
             splitter.basepos += splitter.curpos + 100;
@@ -2021,8 +2016,10 @@
     }
     string ermsg;
     try {
+        statusUpdater()->update(DbIxStatus::DBIXS_FLUSH, "");
         m_ndb->xwdb.commit();
     } XCATCHERROR(ermsg);
+    statusUpdater()->update(DbIxStatus::DBIXS_NONE, "");
     if (!ermsg.empty()) {
         LOGERR("Db::doFlush: flush() failed: " << ermsg << "\n");
         return false;
@@ -2238,8 +2235,10 @@
     // because it doesn't really hurt.
     m_reason.clear();
     try {
+        statusUpdater()->update(DbIxStatus::DBIXS_FLUSH, "");
         m_ndb->xwdb.commit();
     } XCATCHERROR(m_reason);
+    statusUpdater()->update(DbIxStatus::DBIXS_NONE, "");
     if (!m_reason.empty()) {
         LOGERR("Db::purge: 1st flush failed: " << m_reason << "\n");
         return false;
@@ -2286,8 +2285,10 @@
 
     m_reason.clear();
     try {
+        statusUpdater()->update(DbIxStatus::DBIXS_FLUSH, "");
         m_ndb->xwdb.commit();
     } XCATCHERROR(m_reason);
+    statusUpdater()->update(DbIxStatus::DBIXS_NONE, "");
     if (!m_reason.empty()) {
         LOGERR("Db::purge: 2nd flush failed: " << m_reason << "\n");
         return false;
@@ -2556,7 +2557,7 @@
             LOGERR("Db::getSubDocs: xapian error: " << m_reason << "\n");
             return false;
         }
-        if (xit == xdoc.termlist_end()) {
+        if (xit == xdoc.termlist_end() || get_prefix(*xit) != parent_prefix) {
             LOGERR("Db::getSubDocs: parent term not found\n");
             return false;
         }
@@ -2641,7 +2642,7 @@
         LOGERR("Db::getContainerDoc: xapian error: " << m_reason << "\n");
         return false;
     }
-    if (xit == xdoc.termlist_end()) {
+    if (xit == xdoc.termlist_end() || get_prefix(*xit) != parent_prefix) {
         LOGERR("Db::getContainerDoc: parent term not found\n");
         return false;
     }
diff -Nru kio-recoll-1.30.1/rcldb/rcldb.h kio-recoll-1.31.0/rcldb/rcldb.h
--- kio-recoll-1.30.1/rcldb/rcldb.h	2021-02-26 08:43:17.000000000 +0000
+++ kio-recoll-1.31.0/rcldb/rcldb.h	2021-04-24 11:42:51.000000000 +0000
@@ -151,6 +151,20 @@
     return trm.substr(st);
 }
 
+inline string get_prefix(const string& trm)
+{
+    if (!has_prefix(trm))
+        return trm;
+    string::size_type st = 0;
+    if (o_index_stripchars) {
+        st = trm.find_first_not_of("ABCDEFIJKLMNOPQRSTUVWXYZ");
+        return trm.substr(0, st);
+    } else {
+        st = trm.find_last_of(":") + 1;
+        return trm.substr(1, st-2);
+    }
+}
+
 inline string wrap_prefix(const string& pfx) 
 {
     if (o_index_stripchars) {
diff -Nru kio-recoll-1.30.1/rcldb/rclquery.cpp kio-recoll-1.31.0/rcldb/rclquery.cpp
--- kio-recoll-1.30.1/rcldb/rclquery.cpp	2021-03-10 09:38:43.000000000 +0000
+++ kio-recoll-1.31.0/rcldb/rclquery.cpp	2021-04-24 11:45:50.000000000 +0000
@@ -152,8 +152,7 @@
 };
 
 Query::Query(Db *db)
-    : m_nq(new Native(this)), m_db(db), m_sorter(0), m_sortAscending(true),
-      m_collapseDuplicates(false), m_resCnt(-1), m_snipMaxPosWalk(1000000)
+    : m_nq(new Native(this)), m_db(db)
 {
     if (db)
         db->getConf()->getConfParam("snippetMaxPosWalk", &m_snipMaxPosWalk);
@@ -179,6 +178,27 @@
             (m_sortAscending ? "ascending" : "descending") << "\n");
 }
 
+static const string parent_prefix{"F"};
+
+class SubdocDecider : public Xapian::MatchDecider {
+public:
+    SubdocDecider(bool sel) : MatchDecider(), m_select(sel) {}
+    virtual ~SubdocDecider() {}
+    
+    virtual bool operator()(const Xapian::Document &doc) const {
+        bool hasparent{false};
+        try {
+            Xapian::TermIterator xit = doc.termlist_begin();
+            xit.skip_to(wrap_prefix(parent_prefix));
+            hasparent = (xit != doc.termlist_end()) && (get_prefix(*xit) == parent_prefix);
+        } catch (...) {
+        }
+        return hasparent == m_select;
+    }
+
+    bool m_select;
+};
+    
 // Prepare query out of user search data
 bool Query::setQuery(std::shared_ptr sdata)
 {
@@ -199,8 +219,13 @@
         m_reason += sdata->getReason();
         return false;
     }
-
     m_nq->xquery = xq;
+    
+    if (sdata->getSubSpec() == SearchData::SUBDOC_NO) {
+        m_nq->subdecider = new SubdocDecider(false);
+    } else if (sdata->getSubSpec() == SearchData::SUBDOC_YES) {
+        m_nq->subdecider = new SubdocDecider(true);
+    }
 
     string d;
     for (int tries = 0; tries < 2; tries++) {
@@ -361,7 +386,8 @@
         Chrono chron;
         XAPTRY(if (checkatleast == -1)
                    checkatleast = m_db->docCnt();
-               m_nq->xmset = m_nq->xenquire->get_mset(0, qquantum, checkatleast),
+               m_nq->xmset = m_nq->xenquire->get_mset(
+                   0, qquantum, checkatleast, 0, m_nq->subdecider),
                m_db->m_ndb->xrdb, m_reason);
         if (!m_reason.empty()) {
             LOGERR("xenquire->get_mset: exception: " << m_reason << "\n");
@@ -401,10 +427,9 @@
     if (!(xapi >= first && xapi <= last)) {
         LOGDEB("Fetching for first " << xapi << ", count " << qquantum << "\n");
 
-        XAPTRY(m_nq->xmset = m_nq->xenquire->get_mset(xapi, qquantum,  
-                                                      (const Xapian::RSet *)0),
+        XAPTRY(m_nq->xmset = m_nq->xenquire->get_mset(
+                   xapi, qquantum, nullptr, m_nq->subdecider),
                m_db->m_ndb->xrdb, m_reason);
-
         if (!m_reason.empty()) {
             LOGERR("enquire->get_mset: exception: " << m_reason << "\n");
             return false;
diff -Nru kio-recoll-1.30.1/rcldb/rclquery.h kio-recoll-1.31.0/rcldb/rclquery.h
--- kio-recoll-1.30.1/rcldb/rclquery.h	2020-09-05 07:43:16.000000000 +0000
+++ kio-recoll-1.31.0/rcldb/rclquery.h	2021-04-23 14:23:38.000000000 +0000
@@ -139,13 +139,13 @@
 private:
     std::string m_reason; // Error explanation
     Db    *m_db;
-    void  *m_sorter;
+    void  *m_sorter{nullptr};
     std::string m_sortField;
-    bool   m_sortAscending;
-    bool   m_collapseDuplicates;     
-    int    m_resCnt;
+    bool   m_sortAscending{true};
+    bool   m_collapseDuplicates{false};     
+    int    m_resCnt{-1};
     std::shared_ptr m_sd;
-    int    m_snipMaxPosWalk;
+    int    m_snipMaxPosWalk{1000000};
 };
 
 #ifndef NO_NAMESPACES
diff -Nru kio-recoll-1.30.1/rcldb/rclquery_p.h kio-recoll-1.31.0/rcldb/rclquery_p.h
--- kio-recoll-1.30.1/rcldb/rclquery_p.h	2020-08-20 18:59:57.000000000 +0000
+++ kio-recoll-1.31.0/rcldb/rclquery_p.h	2021-04-24 06:24:48.000000000 +0000
@@ -32,24 +32,26 @@
 class Query::Native {
 public:
     // The query I belong to
-    Query *m_q;
+    Query *m_q{nullptr};
     // query descriptor: terms and subqueries joined by operators
     // (or/and etc...)
     Xapian::Query xquery; 
     // Open query descriptor.
-    Xapian::Enquire *xenquire;
+    Xapian::Enquire *xenquire{nullptr};
     // Partial result set
     Xapian::MSet xmset;    
     // Term frequencies for current query. See makeAbstract, setQuery
     std::map  termfreqs; 
-
+    Xapian::MatchDecider *subdecider{nullptr};
+    
     Native(Query *q)
-        : m_q(q), xenquire(0) { }
+        : m_q(q), xenquire(0) {}
     ~Native() {
         clear();
     }
     void clear() {
-        delete xenquire; xenquire = 0;
+        deleteZ(xenquire);
+        deleteZ(subdecider);
         termfreqs.clear();
     }
     /** Return a list of terms which matched for a specific result document */
diff -Nru kio-recoll-1.30.1/rcldb/searchdata.h kio-recoll-1.31.0/rcldb/searchdata.h
--- kio-recoll-1.30.1/rcldb/searchdata.h	2021-02-26 08:43:17.000000000 +0000
+++ kio-recoll-1.31.0/rcldb/searchdata.h	2021-04-24 06:16:20.000000000 +0000
@@ -114,6 +114,17 @@
     void setMinSize(int64_t size) {m_minSize = size;}
     void setMaxSize(int64_t size) {m_maxSize = size;}
 
+    enum SubdocSpec {SUBDOC_ANY = -1, SUBDOC_NO = 0, SUBDOC_YES = 1};
+    void setSubSpec(int spec) {
+        switch (spec) {
+        case SUBDOC_ANY:
+        case SUBDOC_NO:
+        case SUBDOC_YES:
+            m_subspec = spec;
+        }
+    }
+    int getSubSpec() {return m_subspec;}
+    
     /** Set date span for filtering results */
     void setDateSpan(DateInterval *dip) {m_dates = *dip; m_haveDates = true;}
 
@@ -174,12 +185,14 @@
     std::shared_ptr   m_autophrase;
 
     // Special stuff produced by input which looks like a clause but means
-    // something else (date and size specs)
+    // something else (date, size specs, etc.)
     bool           m_haveDates{false};
     DateInterval   m_dates; // Restrict to date interval
     int64_t        m_maxSize{-1};
     int64_t        m_minSize{-1};
-
+    // Filtering for subdocs: -1:any, 0: only free-standing, 1: only subdocs
+    int            m_subspec{SUBDOC_ANY};
+    
     // Printable expanded version of the complete query, retrieved/set
     // from rcldb after the Xapian::setQuery() call
     std::string m_description; 
diff -Nru kio-recoll-1.30.1/VERSION kio-recoll-1.31.0/VERSION
--- kio-recoll-1.30.1/VERSION	2021-04-12 13:01:38.000000000 +0000
+++ kio-recoll-1.31.0/VERSION	2021-04-27 08:13:36.000000000 +0000
@@ -1 +1 @@
-1.30.1
+1.31.0