diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/CHANGES.rst metpx-sarracenia-2.21.08~ubuntu16.04.1/CHANGES.rst --- metpx-sarracenia-2.21.04~ubuntu16.04.1/CHANGES.rst 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/CHANGES.rst 2021-09-02 15:27:55.000000000 +0000 @@ -6,6 +6,15 @@ **git repo** +**2.21.04** + + * bugfix for regression, failure to use BASIC authentication on https sites. + * bugfix running on windows... again for #47, but not resolving #47 + * bugfix #367, stop sftp from doing ls from initializing sender. (for IRIDIUM, from Tyson) + * bugfix... process management in windows. (#47 is about something else, but reported there.) + * bugfix for sender with inflight=None, + * bugfix #360 bearer token support (for some noaa sites.) + **2.20.08post1** * 2nd fix for a different plugin breaking in a different way with onfly_checksum. * sr stop labelling 'status' invocations as missing instances. diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/CHANGES.txt metpx-sarracenia-2.21.08~ubuntu16.04.1/CHANGES.txt --- metpx-sarracenia-2.21.04~ubuntu16.04.1/CHANGES.txt 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/CHANGES.txt 2021-09-02 15:27:55.000000000 +0000 @@ -6,6 +6,15 @@ **git repo** +**2.21.04** + + * bugfix for regression, failure to use BASIC authentication on https sites. + * bugfix running on windows... again for #47, but not resolving #47 + * bugfix #367, stop sftp from doing ls from initializing sender. (for IRIDIUM, from Tyson) + * bugfix... process management in windows. (#47 is about something else, but reported there.) + * bugfix for sender with inflight=None, + * bugfix #360 bearer token support (for some noaa sites.) + **2.20.08post1** * 2nd fix for a different plugin breaking in a different way with onfly_checksum. * sr stop labelling 'status' invocations as missing instances. diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/debian/changelog metpx-sarracenia-2.21.08~ubuntu16.04.1/debian/changelog --- metpx-sarracenia-2.21.04~ubuntu16.04.1/debian/changelog 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/debian/changelog 2021-09-02 15:27:55.000000000 +0000 @@ -1,8 +1,20 @@ -metpx-sarracenia (2.21.04~ubuntu16.04.1) xenial; urgency=low +metpx-sarracenia (2.21.08~ubuntu16.04.1) xenial; urgency=low * Auto build. - -- Supercomputing @ Government of Canada Sat, 17 Apr 2021 08:24:23 +0000 + -- Supercomputing @ Government of Canada Thu, 02 Sep 2021 15:27:55 +0000 + +metpx-sarracenia (2.21.08) UNRELEASED; urgency=medium + + * bugfix #386 not honouring environment variables. + * bugfix #377 not interpreting sum n properly. + * bugfix #374 not interpreting sundew_extension properly. + * bugfix #366 polling of old files when date format on remote server changes. + * bugfix #371 sr_audit should not kill 'edit' sessions as strays. + * hmm.. charset in ftp.. defaults to iso8859-1, changed to utf-8. + noticed caused some issues in newer tests that use accented characters. + + -- Peter Silva Sat, 24 Apr 2021 12:52:19 -0400 metpx-sarracenia (2.21.04) stable; urgency=medium diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/debian/git-build-recipe.manifest metpx-sarracenia-2.21.08~ubuntu16.04.1/debian/git-build-recipe.manifest --- metpx-sarracenia-2.21.04~ubuntu16.04.1/debian/git-build-recipe.manifest 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/debian/git-build-recipe.manifest 2021-09-02 15:27:55.000000000 +0000 @@ -1,2 +1,2 @@ # git-build-recipe format 0.4 deb-version {debupstream} -lp:~ssc-hpc-chp-spc/metpx-sarracenia/+git/trunk git-commit:2c681faea21069f9323f3fba01c2855dc9c09e77 +lp:~ssc-hpc-chp-spc/metpx-sarracenia/+git/trunk git-commit:b791fb53d2b3280dcf6762334d615a72ef99846c diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/doc/fr/sr_poll.1.rst metpx-sarracenia-2.21.08~ubuntu16.04.1/doc/fr/sr_poll.1.rst --- metpx-sarracenia-2.21.04~ubuntu16.04.1/doc/fr/sr_poll.1.rst 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/doc/fr/sr_poll.1.rst 2021-09-02 15:27:55.000000000 +0000 @@ -96,6 +96,7 @@ - **reject (optionnel)** - **chmod (défaut: 0o400)** - **poll_without_vip (défaut: on)** +- **file_time_limit (défaut: 60d)** L'option *filename* peut être utilisée pour définir un renommage global des produits. Ex.. : @@ -161,8 +162,7 @@ moins de charge dans un cas exemplaire.) C´est donc utile de le choisir quand l´efficacité de la configuration global est une enjeu. - - +Par défaut, les fichiers qui sont datés d'il y a plus de deux mois ne seront pas publiés. Il est possible de modifier cette limite de temps en le spécifiant dans une configuration avec *file_date_limit *. Par défaut, ce temps sera en secondes, sauf si préciser autrement, avec 1h, 1d, ou 1w par exemple, pour heure, jour et semaine respectivement. SPÉCIFICATIONS DE PUBLICATION ----------------------------- diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/doc/sr_poll.1.rst metpx-sarracenia-2.21.08~ubuntu16.04.1/doc/sr_poll.1.rst --- metpx-sarracenia-2.21.04~ubuntu16.04.1/doc/sr_poll.1.rst 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/doc/sr_poll.1.rst 2021-09-02 15:27:55.000000000 +0000 @@ -93,6 +93,7 @@ - **reject (optional)** - **chmod (default: 0o400)** - **poll_without_vip (default: True)** +- **file_time_limit (default 60d)** The option *filename* can be used to set a global rename to the products. Ex.: @@ -163,6 +164,7 @@ to *False* (or *off*). This reduces overhead forty-fold in some measured cases. +By default, files that are more than 2 months are not posted. However, this can be modified to any specified time limit in the configurations by using the option *file_time_limit *. By default, seconds are used, but one can specify hours, days or weeks with 1, 1h, 1d, 1w respectively. POSTING SPECIFICATIONS ---------------------- diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/doc/UPGRADING.rst metpx-sarracenia-2.21.08~ubuntu16.04.1/doc/UPGRADING.rst --- metpx-sarracenia-2.21.04~ubuntu16.04.1/doc/UPGRADING.rst 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/doc/UPGRADING.rst 2021-09-02 15:27:55.000000000 +0000 @@ -36,6 +36,36 @@ git --- +2.21.08 +------- + +*CHANGES*: BUGFIXED by this release: https://github.com/MetPX/sarracenia/issues/374 + +sundew_extension was not appended to file name in patter match. This means patterns +such as: + +accept .*.txt$ +reject .*.txt + +may change behaviour, as if the sundew_extension attribute is included in the message, +then it's value will be appended to the url which is evaluated by accept/reject clauses. + + +*CHANGES*: BUG FIXED by this release: https://github.com/MetPX/sarracenia/issues/377 + +it looks like -n (checksum based on file name) was not working as intended... +for a long time. When multiple file versions had different sizes, they would +not be identified as the same file for duplicate_suppression purposes, making +the -n option quite ineffective. Searching through the changelog, it looks like +this has been present for a long time. + +*CHANGES*: BUG FIXED by this release: https://github.com/MetPX/sarracenia/issues/386 + +environment variables as arguments to numeric or truth value arguments are not +honoured in versions older than this. + + + 2.21.04 ------- diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/README.rst metpx-sarracenia-2.21.08~ubuntu16.04.1/README.rst --- metpx-sarracenia-2.21.04~ubuntu16.04.1/README.rst 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/README.rst 2021-09-02 15:27:55.000000000 +0000 @@ -75,7 +75,10 @@ page web / homepage: http://github.com/MetPX/sarracenia -Concepts: `Sarracenia in 10 Minutes Video (anglais seulement) `_ +**Videos:** + +- Concept Overview: `Sarracenia in 10 Minutes Video (anglais seulement) `_ +- v2 Installation Tutorial: `Sarracenia v2 Intro Guide `_ Documentation *master* location: https://github.com/MetPX/sarracenia/blob/master/doc/sr_subscribe.1.rst#documentation diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/__init__.py metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/__init__.py --- metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/__init__.py 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/__init__.py 2021-09-02 15:27:55.000000000 +0000 @@ -25,4 +25,4 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # -__version__ = "2.21.04" +__version__ = "2.21.08" diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_cache.py metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_cache.py --- metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_cache.py 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_cache.py 2021-09-02 15:27:55.000000000 +0000 @@ -69,7 +69,11 @@ self.count = 0 def check(self, key, path, part): - self.logger.debug("sr_cache check basis=%s" % self.cache_basis ) + + if part is None: + self.logger.debug("sr_cache check basis=%s part0=%s key0=%s" % (self.cache_basis, "None", key[0]) ) + else: + self.logger.debug("sr_cache check basis=%s part0=%s key0=%s" % (self.cache_basis, part[0], key[0]) ) # not found self.cache_hit = None @@ -78,7 +82,13 @@ now = nowflt() relpath = self.__get_relpath(path) qpath = urllib.parse.quote(relpath) - value = '%s*%s' % (relpath, part) + + + #override part, when using n because n should be same regardless of size. + if (key[0] == 'n' ) and (part[0] not in [ 'p', 'i' ]): + value = '%s' % (relpath) + else: + value = '%s*%s' % (relpath, part) if key not in self.cache_dict : self.logger.debug("adding a new entry in cache") @@ -89,7 +99,7 @@ self.count += 1 return True - self.logger.debug("sum already in cache: key={}".format(key)) + self.logger.debug("sum already in cache: key value={}".format(value)) kdict = self.cache_dict[key] present = value in kdict kdict[value] = now @@ -195,7 +205,10 @@ parts = value.split('*') path = parts[0] qpath = urllib.parse.quote(path) - part = parts[1] + if len(parts) > 1: + part = parts[1] + else: + part = None if qpath == qdelpath : continue @@ -285,7 +298,11 @@ qpath = words[2] path = urllib.parse.unquote(qpath) part = words[3] - value = '%s*%s' % (path,part) + + if (key[0] == 'n' ) and (part[0] not in [ 'p', 'i' ]): + value = '%s' % path + else: + value = '%s*%s' % (path, part) # skip expired entry diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_config.py metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_config.py --- metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_config.py 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_config.py 2021-09-02 15:27:55.000000000 +0000 @@ -658,7 +658,7 @@ def defaults(self): self.logger.debug("sr_config defaults") - + self.file_time_limit = self.duration_from_str("60d") self.retry_mode = True self.retry_ttl = None @@ -1592,13 +1592,14 @@ self.logger.debug("Masks") self.logger.debug("Masks %s"% self.masks) - + elif words0 =='file_time_limit': + self.file_time_limit = self.duration_from_str(words1) elif words0 in ['accept_unmatched','accept_unmatch','au']: # See: sr_config.7 if (words1 is None) or words[0][0:1] == '-' : self.accept_unmatch = True n = 1 else : - self.accept_unmatch = self.isTrue(words[1]) + self.accept_unmatch = self.isTrue(words1) n = 2 elif words0 in [ 'a', 'action' ]: @@ -1620,7 +1621,7 @@ n = 2 elif words0 == 'batch' : # See: sr_config.7 - self.batch = int(words[1]) + self.batch = int(words1) n = 2 elif words0 in ['base_dir','bd']: # See: sr_config.7 for sr_post.1,sarra,sender,watch @@ -1645,7 +1646,7 @@ self.bind_queue = True n = 1 else : - self.bind_queue = self.isTrue(words[1]) + self.bind_queue = self.isTrue(words1) n = 2 elif words0 in ['broker','b'] : # See: sr_consumer.7 ++ fixme: everywhere, perhaps reduce @@ -1658,7 +1659,7 @@ n = 2 elif words0 == 'blocksize' : # See: sr_config.7 - self.blocksize = self.chunksize_from_str(words[1]) + self.blocksize = self.chunksize_from_str(words1) if self.blocksize == 1: self.parts = '1' ok = self.validate_parts() @@ -1666,7 +1667,7 @@ n = 2 elif words0 == 'bufsize' : # See: sr_config.7 - self.bufsize = int(words[1]) + self.bufsize = int(words1) n = 2 elif words0 in [ 'caching', 'cache', 'no_duplicates', 'noduplicates', 'nd', 'suppress_duplicates', 'sd' ] : # See: sr_post.1 sr_watch.1 @@ -1674,8 +1675,8 @@ self.caching = 300 n = 1 else : - if words[1].isalpha(): - self.caching = self.isTrue(words[1]) + if words1.isalpha(): + self.caching = self.isTrue(words1) if self.caching : self.caching = 300 else : # caching setting is in sec @@ -1703,19 +1704,19 @@ self.cache_stat = True n = 1 else : - self.cache_stat = self.isTrue(words[1]) + self.cache_stat = self.isTrue(words1) n = 2 elif words0 in [ 'chmod', 'default_mode', 'dm']: # See: sr_config.7.rst - self.chmod = int(words[1],8) + self.chmod = int(words1,8) n = 2 elif words0 in [ 'chmod_dir', 'default_dir_mode', 'ddm' ]: # See: sr_config.7.rst - self.chmod_dir = int(words[1],8) + self.chmod_dir = int(words1,8) n = 2 elif words0 in [ 'chmod_log', 'default_log_mode', 'dlm' ]: - self.chmod_log = int(words[1],8) + self.chmod_log = int(words1,8) n = 2 elif words0 in ['cluster','cl','from_cluster','fc']: # See: sr_config.7 @@ -1746,7 +1747,7 @@ if (words1 is None) or words[0][0:1] == '-': self.loglevel = logging.DEBUG n = 1 - elif self.isTrue(words[1]): + elif self.isTrue(words1): self.loglevel = logging.DEBUG n = 2 @@ -1755,7 +1756,7 @@ self.declare_exchange = True n = 1 else : - self.declare_exchange = self.isTrue(words[1]) + self.declare_exchange = self.isTrue(words1) n = 2 elif words0 in ['declare_queue','dq']: # See: sr_config.7 @@ -1763,7 +1764,7 @@ self.declare_queue = True n = 1 else : - self.declare_queue = self.isTrue(words[1]) + self.declare_queue = self.isTrue(words1) n = 2 elif words0 == 'delete': # See: sr_sarra.8 @@ -1771,7 +1772,7 @@ self.delete = True n = 1 else : - self.delete = self.isTrue(words[1]) + self.delete = self.isTrue(words1) n = 2 elif words0 == 'destfn_script': # See: sr_sender(1) @@ -1804,7 +1805,7 @@ self.discard = True n = 1 else : - self.discard = self.isTrue(words[1]) + self.discard = self.isTrue(words1) n = 2 elif words0 in ['document_root','dr']: # See sr_post.1,sarra,sender,watch @@ -1853,7 +1854,7 @@ self.dry_run = True n = 1 else : - self.dry_run = self.isTrue(words[1]) + self.dry_run = self.isTrue(words1) n = 2 elif words0 == 'durable' : # See sr_config.7 ++ @@ -1861,38 +1862,38 @@ self.durable = True n = 1 else : - self.durable = self.isTrue(words[1]) + self.durable = self.isTrue(words1) n = 2 elif words0 in ['events','e']: # See sr_watch.1 i = 0 - if 'deleted' in words[1]: + if 'deleted' in words1: self.logger.warning("deprecated Event spec: please change 'deleted' --> 'delete'") - words[1] = words[1].replace("deleted","delete") + words1 = words1.replace("deleted","delete") - if 'created' in words[1]: + if 'created' in words1: self.logger.warning("deprecated Event spec: please change 'created' --> 'create'") - words[1] = words[1].replace("created","create") + words1 = words1.replace("created","create") - if 'linked' in words[1]: + if 'linked' in words1: self.logger.warning("deprecated Event spec: please change 'linked' --> 'link'") - words[1] = words[1].replace("linked","link") + words1 = words1.replace("linked","link") - if 'modified' in words[1]: + if 'modified' in words1: self.logger.warning("deprecated event spec: please change 'modified' --> 'modify'") - words[1] = words[1].replace("modified","modify") + words1 = words1.replace("modified","modify") - if 'create' in words[1] : i = i + 1 - if 'delete' in words[1] : i = i + 1 - if 'link' in words[1] : i = i + 1 - if 'modify' in words[1] : i = i + 1 - if 'move' in words[1] : i = i + 1 + if 'create' in words1 : i = i + 1 + if 'delete' in words1 : i = i + 1 + if 'link' in words1 : i = i + 1 + if 'modify' in words1 : i = i + 1 + if 'move' in words1 : i = i + 1 - if i < len(words[1].split(',')) : - self.logger.error("events invalid (%s)" % words[1]) + if i < len(words1.split(',')) : + self.logger.error("events invalid (%s)" % words1) needexit = True - self.events = words[1] + self.events = words1 n = 2 elif words0 in ['exchange','ex'] : # See: sr_config.7 ++ everywhere fixme? @@ -1904,7 +1905,7 @@ self.exchange_split = True n = 1 else : - self.exchange_split = self.isTrue(words[1]) + self.exchange_split = self.isTrue(words1) n = 2 elif words0 in ['exchange_suffix'] : # FIXME: sr_config.7 ++ everywhere fixme? @@ -1918,11 +1919,11 @@ # rabbitmq setting is in millisec / user in secs self.expire = int(self.duration_from_str(words1,'ms')) if self.expire < 300000 : - self.logger.warning("expire setting (%s) may cause problem...too low" % words[1]) + self.logger.warning("expire setting (%s) may cause problem...too low" % words1) n = 2 elif words0 == 'filename': # See: sr_poll.1, sr_sender.1 - self.currentFileOption = words[1] + self.currentFileOption = words1 n = 2 elif words0 in ['realpath_filter','fr']: # FIXME: MG new @@ -1930,11 +1931,11 @@ self.realpath_filter = True n = 1 else : - self.realpath_filter = self.isTrue(words[1]) + self.realpath_filter = self.isTrue(words1) n = 2 elif words0 in [ 'flatten' ]: # See: sr_poll.1, sr_sender.1 - self.flatten = words[1] + self.flatten = words1 n = 2 elif words0 in ['follow_symlinks','fs']: # See: sr_post.1, sr_watch.1 @@ -1942,7 +1943,7 @@ self.follow_symlinks = True n = 1 else : - self.follow_symlinks = self.isTrue(words[1]) + self.follow_symlinks = self.isTrue(words1) n = 2 elif words0 in ['force_polling','fp']: # See: sr_post.1, sr_watch.1 @@ -1950,7 +1951,7 @@ self.force_polling = True n = 1 else : - self.force_polling = self.isTrue(words[1]) + self.force_polling = self.isTrue(words1) n = 2 elif words0 in ['header']: # See: sr_config.7 @@ -1998,7 +1999,7 @@ os._exit(0) elif words0 in ['hostname']: # See: dd_subscribe (obsolete option...ok) - self.hostname = words[1] + self.hostname = words1 n = 2 elif words0 in ['inline','inl','content' ]: # See: sr_config.7 @@ -2006,7 +2007,7 @@ self.inline = True n = 1 else : - self.inline = self.isTrue(words[1]) + self.inline = self.isTrue(words1) n = 2 elif words0 in ['inline_encoding','inlenc','content_encoding' ]: # See: sr_config.7 @@ -2020,7 +2021,7 @@ n = 2 elif words0 in ['inline_max','imx', 'content_max' ]: # See: sr_config.7 - self.inline_max = int(words[1]) + self.inline_max = int(words1) n = 2 elif words0 in ['inplace','in','assemble']: # See: sr_sarra.8, sr_post.1, sr_watch.1 @@ -2028,33 +2029,33 @@ self.inplace = True n = 1 else : - self.inplace = self.isTrue(words[1]) + self.inplace = self.isTrue(words1) n = 2 elif words0 in ['instance', 'instances','i']: # See: sr_config.7 - self.nbr_instances = int(words[1]) + self.nbr_instances = int(words1) n = 2 elif words0 == 'interface': # See: sr_poll, sr_winnow self.logger.warning("deprecated *interface* option no longer has any effect, vip is enough." ) - self.interface = words[1] + self.interface = words1 n = 2 elif words0 == 'kbytes_ps': # See: sr_sender - self.kbytes_ps = int(words[1]) + self.kbytes_ps = int(words1) n = 2 elif words0 in ['lock','inflight']: # See: sr_config.7, sr_subscribe.1 if words0 in [ 'lock' ]: # FIXME: remove support in 2019. self.logger.warning( "Deprecated option. Please use *inflight* instead of *lock*" ) - if words[1].lower() in [ 'none' ]: + if words1.lower() in [ 'none' ]: self.inflight=None - elif words[1][0].isnumeric() : + elif words1[0].isnumeric() : self.inflight = self.duration_from_str(words1,'s') if self.inflight <= 1 : self.inflight = None else: - self.inflight = words[1] + self.inflight = words1 n = 2 elif words0 in [ 'log_reject' ]: # See: sr_sarra.8 @@ -2062,12 +2063,12 @@ self.log_reject = True n = 1 else : - self.log_reject = self.isTrue(words[1]) + self.log_reject = self.isTrue(words1) n = 2 elif words0 == 'ls_file_index': # FIX ME to document... position of file in ls # use when space in filename is expected - self.ls_file_index = int(words[1]) + self.ls_file_index = int(words1) n = 2 @@ -2076,7 +2077,7 @@ self.pipe = True n = 1 else : - self.pipe = self.isTrue(words[1]) + self.pipe = self.isTrue(words1) n = 2 elif words0 == 'restore' : # See: sr_config.7 @@ -2085,7 +2086,7 @@ self.restore = True n = 1 else : - self.restore = self.isTrue(words[1]) + self.restore = self.isTrue(words1) n = 2 elif words0 in ['restore_to_queue', 'restore2queue', 'r2q', 'rq', 'post_queue']: @@ -2099,7 +2100,7 @@ self.report_daemons = True n = 1 else : - self.report_daemons = self.isTrue(words[1]) + self.report_daemons = self.isTrue(words1) n = 2 elif words0 in ['report_exchange', 'lx', 'le'] : # See: sr_config.7 ++ everywhere fixme? @@ -2151,7 +2152,7 @@ n = 2 elif words0 == 'max_queue_size': # See: sr_audit.8 (sr_config also) - self.max_queue_size = int(words[1]) + self.max_queue_size = int(words1) n = 2 elif words0 in [ 'message-ttl', 'message_ttl' ]: # See: sr_consumer.7 @@ -2161,7 +2162,7 @@ # rabbitmq setting is in millisec self.message_ttl = int(self.duration_from_str(words1,'ms')) if self.message_ttl < 300000 : - self.logger.warning("message_ttl setting (%s) may cause problem...too low" % words[1]) + self.logger.warning("message_ttl setting (%s) may cause problem...too low" % words1) n = 2 elif words0 == 'mirror': # See: sr_config.7 @@ -2169,7 +2170,7 @@ self.mirror = True n = 1 else : - self.mirror = self.isTrue(words[1]) + self.mirror = self.isTrue(words1) n = 2 elif words0 == 'move': # See: sr_post.1 @@ -2182,7 +2183,7 @@ # there are N instances asked to start each one having its own number (no) # -no 1, -no 2, ... -no N elif words0 == 'no': - self.no = int(words[1]) + self.no = int(words1) n = 2 elif words0 in ['notify_only','n','no_download']: # See: sr_subscribe.1 @@ -2190,7 +2191,7 @@ self.notify_only = True n = 1 else: - self.notify_only = self.isTrue(words[1]) + self.notify_only = self.isTrue(words1) n = 2 elif words0 == 'on_data': # See: sr_config.7, sr_sarra,shovel,subscribe @@ -2278,11 +2279,11 @@ self.overwrite = True n = 1 else : - self.overwrite = self.isTrue(words[1]) + self.overwrite = self.isTrue(words1) n = 2 elif words0 == 'parts': # See: sr_poll.1, sr_watch.1 - self.parts = words[1] + self.parts = words1 ok = self.validate_parts() if not ok : needexit = True n = 2 @@ -2373,7 +2374,7 @@ self.post_on_start = True n = 1 else : - self.post_on_start = self.isTrue(words[1]) + self.post_on_start = self.isTrue(words1) n = 2 elif words0 in ['post_topic_prefix', 'ptp' ]: # FIXME: sr_sarra,sender,shovel,winnow @@ -2393,7 +2394,7 @@ self.poll_without_vip = True n = 1 else : - self.poll_without_vip = self.isTrue(words[1]) + self.poll_without_vip = self.isTrue(words1) n = 2 elif words0 == 'prefetch': # See: sr_consumer.1 (Nbr of prefetch message when queue is shared) @@ -2405,7 +2406,7 @@ self.preserve_mode = True n = 1 else : - self.preserve_mode = self.isTrue(words[1]) + self.preserve_mode = self.isTrue(words1) n = 2 elif words0 in ['preserve_time','pt'] : # See: sr_config.7 @@ -2413,7 +2414,7 @@ self.preserve_time = True n = 1 else : - self.preserve_time = self.isTrue(words[1]) + self.preserve_time = self.isTrue(words1) n = 2 elif words0 == 'pump': # See: sr_audit.1 (give pump hints or setting errors) @@ -2421,7 +2422,7 @@ self.pump_flag = True n = 1 else : - self.pump_flag = self.isTrue(words[1]) + self.pump_flag = self.isTrue(words1) n = 2 elif words0 in ['queue', 'queue_name','qn'] : # See: sr_config.7, sender, shovel, sub, winnow too much? @@ -2438,7 +2439,7 @@ self.randomize = True n = 1 else : - self.randomize = self.isTrue(words[1]) + self.randomize = self.isTrue(words1) n = 2 elif words0 in ['realpath_post','realpath','real']: # See: sr_post.1, sr_watch.1 @@ -2446,7 +2447,7 @@ self.realpath_post = True n = 1 else : - self.realpath_post = self.isTrue(words[1]) + self.realpath_post = self.isTrue(words1) n = 2 elif words0 in ['reconnect','rr']: # See: sr_post.1, sr_watch.1 @@ -2454,11 +2455,11 @@ self.reconnect = True n = 1 else : - self.reconnect = self.isTrue(words[1]) + self.reconnect = self.isTrue(words1) n = 2 elif words0 in ['remote_config_url']: # See: sr_config.7 - self.remote_config_url = words[1] + self.remote_config_url = words1 n = 2 elif words0 in ['rename','rn']: # See: sr_poll, sarra, sender, sub, watch? @@ -2470,7 +2471,7 @@ self.reportback = True n = 1 else : - self.reportback = self.isTrue(words[1]) + self.reportback = self.isTrue(words1) n = 2 elif words0 in ['reset']: # See: sr_consumer.1 @@ -2478,7 +2479,7 @@ self.reset = True n = 1 else : - self.reset = self.isTrue(words[1]) + self.reset = self.isTrue(words1) n = 2 elif words0 in [ 'retry', 'retry_mode']: # See: sr_consumer.1 @@ -2486,7 +2487,7 @@ self.retry_mode = True n = 1 else : - self.retry_mode = self.isTrue(words[1]) + self.retry_mode = self.isTrue(words1) n = 2 elif words0 in ['retry_ttl']: # FIXME to be documented @@ -2497,7 +2498,7 @@ n = 2 elif words0 in [ 'role', 'declare' ]: # See: sr_audit.1 - item = words[1].lower() + item = words1.lower() if words0 in [ 'role' ]: self.logger.warning("role option deprecated, please replace with 'declare'" ) @@ -2529,11 +2530,11 @@ self.save = True n = 1 else : - self.save = self.isTrue(words[1]) + self.save = self.isTrue(words1) n = 2 elif words0 in [ 'save_file', 'sf' ]: # FIXMEFIXME - self.save_file = words[1] + self.save_file = words1 n = 2 elif words0 in ['set_passwords']: # See: sr_consumer.1 @@ -2541,7 +2542,7 @@ self.set_passwords = True n = 1 else : - self.set_passwords = self.isTrue(words[1]) + self.set_passwords = self.isTrue(words1) n = 2 elif words0 == 'sleep': # See: sr_audit.8 sr_poll.1 @@ -2551,7 +2552,7 @@ n = 2 elif words0 == 'source': # See: sr_post.1 sr_watch.1 - self.source = words[1] + self.source = words1 n = 2 elif words0 in ['source_from_exchange','sfe']: # See: sr_sarra.8 @@ -2559,7 +2560,7 @@ self.source_from_exchange = True n = 1 else : - self.source_from_exchange = self.isTrue(words[1]) + self.source_from_exchange = self.isTrue(words1) n = 2 elif words0 == 'statehost': # MG FIXME to be documented somewhere ??? @@ -2571,7 +2572,7 @@ self.hostform = words1.lower() n = 2 else: - if not self.isTrue(words[1]): + if not self.isTrue(words1): self.statehost = False n = 2 @@ -2611,7 +2612,7 @@ n = 2 elif words0 == 'sum': # See: sr_config.7 - self.sumflg = words[1] + self.sumflg = words1 ok = self.validate_sum() if not ok : self.logger.error("unknown checksum specified: %s, should be one of: %s or z" % ( self.sumflg, ', '.join(self.sumalgos.keys()) ) ) @@ -2674,7 +2675,7 @@ self.use_amqplib = True n = 1 elif not self.use_pika and amqplib_available: - self.use_amqplib = self.isTrue(words[1]) + self.use_amqplib = self.isTrue(words1) n = 2 else: n = 2 @@ -2684,7 +2685,7 @@ self.use_pika = True n = 1 elif not self.use_amqplib and pika_available: - self.use_pika = self.isTrue(words[1]) + self.use_pika = self.isTrue(words1) n = 2 else: n = 2 @@ -2694,11 +2695,11 @@ self.users_flag = True n = 1 else : - self.users_flag = self.isTrue(words[1]) + self.users_flag = self.isTrue(words1) n = 2 elif words0 == 'vip': # See: sr_poll.1, sr_winnow.1 - self.vip = words[1] + self.vip = words1 n = 2 elif words0 in [ 'windows_run', 'wr' ] : # See: sr_post.1 sr_watch.1 @@ -2716,7 +2717,7 @@ xattr_disabled = True n = 1 else : - xattr_disabled = self.isTrue(words[1]) + xattr_disabled = self.isTrue(words1) n = 2 if xattr_disabled: self.logger.info("checksum caching in file attributes has been disabled") diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_consumer.py metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_consumer.py --- metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_consumer.py 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_consumer.py 2021-09-02 15:27:55.000000000 +0000 @@ -205,7 +205,7 @@ self.logger.debug("sr_consumer, path being matched: %s " % ( urlstr ) ) - if not self.parent.isMatchingPattern(self.msg.urlstr,self.accept_unmatch) : + if not self.parent.isMatchingPattern(urlstr,self.accept_unmatch) : self.logger.debug("Rejected by accept/reject options") return False,self.msg diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_ftp.py metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_ftp.py --- metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_ftp.py 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_ftp.py 2021-09-02 15:27:55.000000000 +0000 @@ -187,6 +187,8 @@ if not self.tls : ftp = ftplib.FTP() + # in 2021, better to assume utf-8 rather than iso8859-1 + ftp.encoding='utf-8' ftp.connect(self.host,self.port,timeout=expire) ftp.login(self.user, self.password) else : diff -Nru metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_poll.py metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_poll.py --- metpx-sarracenia-2.21.04~ubuntu16.04.1/sarra/sr_poll.py 2021-04-17 08:24:23.000000000 +0000 +++ metpx-sarracenia-2.21.08~ubuntu16.04.1/sarra/sr_poll.py 2021-09-02 15:27:55.000000000 +0000 @@ -31,7 +31,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # -#============================================================ +# ============================================================ # usage example # # sr_poll [options] [config] [foreground|start|stop|restart|reload|status|cleanup|setup] @@ -62,90 +62,135 @@ # to = message.headers['to_clusters'] MANDATORY # # option to only post... no download ... or only post -#============================================================ +# ============================================================ # -import os,sys,time +import os, sys, time +import datetime -#============================================================ +# ============================================================ # DECLARE TRICK for false self.poster from collections import * -#============================================================ -try : - from sr_cache import * - from sr_file import * - from sr_ftp import * - from sr_http import * - from sr_message import * - from sr_post import * - from sr_util import * -except : - from sarra.sr_cache import * - from sarra.sr_file import * - from sarra.sr_ftp import * - from sarra.sr_http import * - from sarra.sr_message import * - from sarra.sr_post import * - from sarra.sr_util import * +# ============================================================ +try: + from sr_cache import * + from sr_file import * + from sr_ftp import * + from sr_http import * + from sr_message import * + from sr_post import * + from sr_util import * +except: + from sarra.sr_cache import * + from sarra.sr_file import * + from sarra.sr_ftp import * + from sarra.sr_http import * + from sarra.sr_message import * + from sarra.sr_post import * + from sarra.sr_util import * class sr_poll(sr_post): def cd(self, path): - try : - self.dest.cd(path) - return True - except : - self.logger.warning("sr_poll/cd: could not cd to directory %s" % path ) + try: + self.dest.cd(path) + return True + except: + self.logger.warning("sr_poll/cd: could not cd to directory %s" % path) return False def check(self): - if self.config_name == None : return + if self.config_name == None: return # check destination self.details = None - if self.destination != None : - ok, self.details = self.credentials.get(self.destination) + if self.destination != None: + ok, self.details = self.credentials.get(self.destination) - if self.destination == None or self.details == None : - self.logger.error("destination option incorrect or missing\n") - self.help() - sys.exit(1) - - if self.post_base_url == None : - self.post_base_url = self.details.url.geturl() - if self.post_base_url[-1] != '/' : self.post_base_url += '/' - if self.post_base_url.startswith('file:'): self.post_base_url = 'file:' - if self.details.url.password : - self.post_base_url = self.post_base_url.replace(':'+self.details.url.password,'') + if self.destination == None or self.details == None: + self.logger.error("destination option incorrect or missing\n") + self.help() + sys.exit(1) + + if self.post_base_url == None: + self.post_base_url = self.details.url.geturl() + if self.post_base_url[-1] != '/': self.post_base_url += '/' + if self.post_base_url.startswith('file:'): self.post_base_url = 'file:' + if self.details.url.password: + self.post_base_url = self.post_base_url.replace(':' + self.details.url.password, '') sr_post.check(self) - self.sleeping = False - self.connected = False + self.sleeping = False + self.connected = False # rebuild mask as pulls instructions # pulls[directory] = [mask1,mask2...] - self.pulls = {} + self.pulls = {} for mask in self.masks: pattern, maskDir, maskFileOption, mask_regexp, accepting, mirror, strip, pstrip, flatten = mask self.logger.debug(mask) - if not maskDir in self.pulls : - self.pulls[maskDir] = [] + if not maskDir in self.pulls: + self.pulls[maskDir] = [] self.pulls[maskDir].append(mask) + + def _file_date_exceed_limit(self, date, time_limit): + """comments: + This method compares today's date to the file's date by creating a date time object + Three formats are acceptd so far, more can be added if needed (format on https://strftime.org/ ) + Files with future dates are processed as long as the (future date - todays date) is < time_limit. + FIXME: french input like Fev will not work - only Feb is accepted for the month + If year is not provided, this means that the file is < 6 months old, so depending on todays date, + assign appropriate year (for jan-jun -> assign prev year, for jul-dec assign current year) + Note: is it possible for a file to be more than 6 months old and have the format Mo Day TIME ? (problematic) + """ + time_limit=int(time_limit) + current_date = datetime.datetime.now() + try: + date_temp = datetime.datetime.strptime(date, '%d %b %H:%M') + if date_temp.month - current_date.month >= 6: + file_date = date_temp.replace(year=(current_date.year - 1)) + else: + file_date = date_temp.replace(year=current_date.year) + self.logger.info("File date is: " + str(file_date) + + " > File is " + str(abs((file_date - current_date).seconds))+" seconds old") + return abs((file_date - current_date).seconds) < time_limit + except Exception as e: + try: + date_temp = datetime.datetime.strptime(date, '%b %d %H:%M') + if date_temp.month - current_date.month >= 6: + file_date = date_temp.replace(year=(current_date.year - 1)) + else: + file_date = date_temp.replace(year=current_date.year) + self.logger.info("File date is: " + str(file_date) + + " > File is " + str(abs((file_date - current_date).seconds))+" seconds old") + return abs((file_date - current_date).seconds) < time_limit + except Exception as e: + try: + file_date = datetime.datetime.strptime(date, '%b %d %Y') + self.logger.info("File date is: " + str(file_date) + + " > File is " + str(abs((file_date - current_date).seconds)) + " seconds old") + return abs((file_date - current_date).seconds) < time_limit + except Exception as e: + warning_msg = str(e) + print("%s" % warning_msg) + return False + # find differences between current ls and last ls # only the newer or modified files will be kept... - def differ_ls_file(self,ls,lspath): + + def differ_ls_file(self, ls, lspath): # get new list and description @@ -153,34 +198,47 @@ # get old list and description - old_ls = self.load_ls_file(lspath) + old_ls = self.load_ls_file(lspath) # compare - filelst = [] - desclst = {} - - for f in new_lst : - #self.logger.debug("checking %s (%s)" % (f,ls[f])) - - # keep a newer entry - if not f in old_ls: - #self.logger.debug("IS NEW %s" % f) - filelst.append(f) - desclst[f] = ls[f] - continue - - # keep a modified entry - if ls[f] != old_ls[f] : - #self.logger.debug("IS DIFFERENT %s from (%s,%s)" % (f,old_ls[f],ls[f])) - filelst.append(f) - desclst[f] = ls[f] - continue + filelst = [] + desclst = {} - #self.logger.debug("IS IDENTICAL %s" % f) - - return filelst,desclst + for f in new_lst: + # self.logger.debug("checking %s (%s)" % (f,ls[f])) + try: + str1 = ls[f] + str2 = str1.split() + # specify input for this routine. + # ls[f] format controlled by online plugin (line_mode.py) + # this format could change depending on plugin + # line_mode.py format "-rwxrwxr-x 1 1000 1000 8123 24 Mar 22:54 2017-03-25-0254-CL2D-AUTO-minute-swob.xml" + date = str2[5] + " " + str2[6] + " " + str2[7] + if self._file_date_exceed_limit(date, self.file_time_limit): + self.logger.info("File should be processed") + # execute rest of code + # keep a newer entry + if not f in old_ls: + # self.logger.debug("IS NEW %s" % f) + filelst.append(f) + desclst[f] = ls[f] + continue + + # keep a modified entry + if ls[f] != old_ls[f]: + # self.logger.debug("IS DIFFERENT %s from (%s,%s)" % (f,old_ls[f],ls[f])) + filelst.append(f) + desclst[f] = ls[f] + continue + else: + self.logger.info("File should be skipped") + # ignore rest of code and re iterate + except: + pass + # self.logger.debug("IS IDENTICAL %s" % f) + return filelst, desclst # ============= # __do_poll__ @@ -188,46 +246,52 @@ def __do_poll__(self): - scheme = self.details.url.scheme + scheme = self.details.url.scheme # try registered do_poll first... might overload defaults try: - if scheme in self.do_polls : - self.logger.debug("using registered do_poll for %s" % scheme) - do_poll = self.do_polls[scheme] - ok = do_poll(self) - return ok - except: pass + if scheme in self.do_polls: + self.logger.debug("using registered do_poll for %s" % scheme) + do_poll = self.do_polls[scheme] + ok = do_poll(self) + return ok + except: + pass # try supported hardcoded download self.dest = None - if scheme == 'file' : self.dest = sr_file(self) - elif scheme in ['ftp','ftps'] : self.dest = sr_ftp(self) - elif scheme in ['http','https']: self.dest = sr_http(self) - elif scheme == 'sftp' : - try : from sr_sftp import sr_sftp - except : from sarra.sr_sftp import sr_sftp - self.dest = sr_sftp(self) + if scheme == 'file': + self.dest = sr_file(self) + elif scheme in ['ftp', 'ftps']: + self.dest = sr_ftp(self) + elif scheme in ['http', 'https']: + self.dest = sr_http(self) + elif scheme == 'sftp': + try: + from sr_sftp import sr_sftp + except: + from sarra.sr_sftp import sr_sftp + self.dest = sr_sftp(self) # user defined poll scripts # if many are configured, this one is the last one in config - if self.do_poll : - ok = self.do_poll(self) - return ok + if self.do_poll: + ok = self.do_poll(self) + return ok # standard poll to post new urls - if self.dest != None : - ok = self.post_new_urls() - return ok + if self.dest != None: + ok = self.post_new_urls() + return ok # user defined poll scripts # if many are configured, this one is the last one in config - #if self.do_poll : + # if self.do_poll : # ok = self.do_poll(self) # return ok @@ -238,94 +302,94 @@ return False def help(self): - print("Usage: %s [OPTIONS] configfile [add|cleanup|declare|disable|edit|enable|foreground|remove|start|stop|restart|reload|setup|status]\n" % self.program_name ) - print("version: %s \n" % sarra.__version__ ) + print( + "Usage: %s [OPTIONS] configfile [add|cleanup|declare|disable|edit|enable|foreground|remove|start|stop|restart|reload|setup|status]\n" % self.program_name) + print("version: %s \n" % sarra.__version__) print("\n\tPoll a remote server to produce announcements of new files appearing there\n" + - "\npoll.conf file settings, MANDATORY ones must be set for a valid configuration:\n" + - "\nAMQP broker settings:\n" + - "\tpost_broker amqp{s}://:@[:port]/\n" + - "\t\t(default: amqp://anonymous:anonymous@dd.weather.gc.ca/ ) \n" + - "\nAMQP Queue bindings:\n" + - "\tpost_exchange (default: xreport for feeders, xs_)\n" + - "\ttopic_prefix (invariant prefix, currently v02.report)\n" + - "\tsubtopic (MANDATORY)\n" + - "\t\t = .....\n" + - "\t\t\t* single directory wildcard (matches one directory)\n" + - "\t\t\t# wildcard (matches rest)\n" + - "\nAMQP Queue settings:\n" + - "\tdurable (default: False)\n" + - "\texpire (default: None)\n" + - "\tmessage-ttl (default: None)\n" + - "\tqueue_name (default: program set it for you)\n" + - "\ttls_rigour (lax|medium|high) (default: medium)\n" + - "\t SSL strictness, use lax for expired certificates.\n" + - "\nProcessing:\n" + - "\tdo_line