diff -Nru biomaj3-3.1.18/biomaj/bank.py biomaj3-3.1.22/biomaj/bank.py --- biomaj3-3.1.18/biomaj/bank.py 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/biomaj/bank.py 2022-05-04 06:55:44.000000000 +0000 @@ -182,7 +182,7 @@ str(last_update), str(release)]) # Bank production info header - prod_info.append(["Session", "Remote release", "Release", "Directory", "Freeze", "Format(s)"]) + prod_info.append(["Session", "Remote release", "Release", "Directory", "Freeze", "Format(s)", "Log file"]) for prod in _bank['production']: data_dir = self.config.get('data.dir') dir_version = self.config.get('dir.version') @@ -205,7 +205,9 @@ prod['release'], release_dir, 'yes' if 'freeze' in prod and prod['freeze'] else 'no', - formats]) + formats, + prod.get('log_file', '') + ]) # Bank pending info header if 'pending' in _bank and len(_bank['pending']) > 0: pend_info.append(["Pending release", "Last run"]) @@ -579,6 +581,7 @@ 'data_dir': self.session._session['data_dir'], 'dir_version': self.session._session['dir_version'], 'prod_dir': prod_dir, + 'log_file': self.session._session['log_file'], 'freeze': False} self.bank['production'].append(production) self.banks.update({'name': self.name}, diff -Nru biomaj3-3.1.18/biomaj/process/processfactory.py biomaj3-3.1.22/biomaj/process/processfactory.py --- biomaj3-3.1.18/biomaj/process/processfactory.py 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/biomaj/process/processfactory.py 2022-05-04 06:55:44.000000000 +0000 @@ -66,7 +66,7 @@ kill_received = True for t in running_th: t.kill_received = True - running_th = [t.join(1000) for t in running_th if t is not None and t.isAlive()] + running_th = [t.join(1000) for t in running_th if t is not None and t.is_alive()] except KeyboardInterrupt: logging.warn("Ctrl-c received! Sending kill to threads...") logging.warn("Running tasks will continue and process will stop.") diff -Nru biomaj3-3.1.18/biomaj/workflow.py biomaj3-3.1.22/biomaj/workflow.py --- biomaj3-3.1.18/biomaj/workflow.py 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/biomaj/workflow.py 2022-05-04 06:55:44.000000000 +0000 @@ -26,6 +26,8 @@ from biomaj_zipkin.zipkin import Zipkin from yapsy.PluginManager import PluginManager +from packaging.version import parse + class Workflow(object): """ @@ -516,25 +518,16 @@ Try to find most release from releases input array ''' release = releases[0] - releaseElts = re.split(r'\.|-', release) + release_version = parse(release) logging.debug('found a release %s' % (release)) for rel in releases: if rel == release: continue logging.debug('compare next release %s' % (rel)) - relElts = re.split(r'\.|-', rel) - index = 0 - for relElt in relElts: - logging.debug("compare release major,minor,etc. : %s >? %s" % (relElt, releaseElts[index])) - try: - if int(relElt) > int(releaseElts[index]): - release = rel - logging.debug("found newer release %s" % (rel)) - break - except ValueError: - pass - finally: - index += 1 + next_release = parse(rel) + if next_release > release_version: + release = rel + release_version = next_release return release def wf_release(self): @@ -1663,9 +1656,7 @@ nb_try = 1 origFile = self.session.get_offline_directory() + '/' + file['save_as'] is_archive = False - if origFile.endswith('.tar.gz'): - is_archive = True - elif origFile.endswith('.tar'): + if origFile.endswith(('.tar.gz', '.tar', '.tgz')): is_archive = True elif origFile.endswith('.bz2'): is_archive = True diff -Nru biomaj3-3.1.18/CHANGES.txt biomaj3-3.1.22/CHANGES.txt --- biomaj3-3.1.18/CHANGES.txt 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/CHANGES.txt 2022-05-04 06:55:44.000000000 +0000 @@ -1,3 +1,13 @@ +3.1.21: + Freeze pymongo to 3.12.3 (4.x breaks) + Change isAlive() which is deprecated in python 3.9 to is_alive + +3.1.20: + Follow-up of #127 to get last release in file (refactor and bug fix) +3.1.19: + Add tgz archive support + Add log file info to production info + #126 Issue with getting last release in file 3.1.18: Python 3 support only If multiple files match release.file, take most recent one diff -Nru biomaj3-3.1.18/debian/changelog biomaj3-3.1.22/debian/changelog --- biomaj3-3.1.18/debian/changelog 2021-02-19 10:25:44.000000000 +0000 +++ biomaj3-3.1.22/debian/changelog 2022-06-30 14:48:27.000000000 +0000 @@ -1,7 +1,20 @@ +biomaj3 (3.1.22-1) unstable; urgency=medium + + * Team upload. + * New upstream version + * Standards-Version: 4.6.1 (routine-update) + * Remove trailing whitespace in debian/changelog (routine-update) + * Re-enable influxdb which was fixed + * Enable at least superfluous autopkgtest + * Fix lintian-overrides + * DEP3 + + -- Andreas Tille Thu, 30 Jun 2022 16:48:27 +0200 + biomaj3 (3.1.18-2) unstable; urgency=medium * Remove drmaa support (optional feature) due to drmaa package - migration issue on 0.7.9-2 + migration issue on 0.7.9-2 -- Olivier Sallou Fri, 19 Feb 2021 10:25:44 +0000 diff -Nru biomaj3-3.1.18/debian/control biomaj3-3.1.22/debian/control --- biomaj3-3.1.18/debian/control 2021-02-19 10:25:44.000000000 +0000 +++ biomaj3-3.1.22/debian/control 2022-06-30 14:48:27.000000000 +0000 @@ -20,7 +20,7 @@ python3-biomaj3-download, python3-biomaj3-user, python3-biomaj3-process -Standards-Version: 4.5.1 +Standards-Version: 4.6.1 Vcs-Browser: https://salsa.debian.org/med-team/biomaj3 Vcs-Git: https://salsa.debian.org/med-team/biomaj3.git Homepage: https://github.com/genouest/biomaj @@ -35,7 +35,6 @@ Suggests: ${python3:Suggests}, python3-biomaj3-cli, python3-gunicorn, - mongodb, redis-server Description: BioMAJ workflow management library BioMAJ downloads remote data banks, checks their status and applies diff -Nru biomaj3-3.1.18/debian/lintian-overrides biomaj3-3.1.22/debian/lintian-overrides --- biomaj3-3.1.18/debian/lintian-overrides 2021-02-19 10:25:44.000000000 +0000 +++ biomaj3-3.1.22/debian/lintian-overrides 2022-06-30 14:48:27.000000000 +0000 @@ -1,2 +1,2 @@ # see https://lists.debian.org/debian-med/2018/06/msg00043.html -python3-biomaj3: script-with-language-extension usr/bin/*.* +python3-biomaj3: script-with-language-extension diff -Nru biomaj3-3.1.18/debian/patches/fix_remove_drmaa.patch biomaj3-3.1.22/debian/patches/fix_remove_drmaa.patch --- biomaj3-3.1.18/debian/patches/fix_remove_drmaa.patch 2021-02-19 10:25:44.000000000 +0000 +++ biomaj3-3.1.22/debian/patches/fix_remove_drmaa.patch 2022-06-30 14:48:27.000000000 +0000 @@ -1,17 +1,21 @@ +Author: Olivier Sallou +Last-Update: 2021-02-19 10:27:50 +0000 +Description: drmaa seems to be broken so disable this feature + --- a/docs/conf.py +++ b/docs/conf.py -@@ -22,7 +22,7 @@ +@@ -22,7 +22,7 @@ class Mock(MagicMock): def __getattr__(cls, name): return Mock() --MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch', 'drmaa', -+MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch', +-MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch', 'drmaa', 'influxdb', ++MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch', 'influxdb', 'biomaj_download', 'biomaj_download.downloadclient', 'biomaj_download.download', --- a/docs/requirements.txt +++ b/docs/requirements.txt -@@ -4,7 +4,6 @@ +@@ -4,7 +4,6 @@ pymongo==3.2 tabulate ldap3 py-bcrypt @@ -21,7 +25,7 @@ biomaj_core --- a/requirements.txt +++ b/requirements.txt -@@ -9,7 +9,6 @@ +@@ -9,7 +9,6 @@ pymongo==3.12.3 pycurl tabulate py-bcrypt @@ -31,8 +35,8 @@ requests --- a/setup.py +++ b/setup.py -@@ -65,7 +65,6 @@ - 'pymongo>=3.2', +@@ -65,7 +65,6 @@ config = { + 'pymongo >=3.2, <4', 'pycurl', 'py-bcrypt', - 'drmaa', diff -Nru biomaj3-3.1.18/debian/patches/fix_remove_influxdb.patch biomaj3-3.1.22/debian/patches/fix_remove_influxdb.patch --- biomaj3-3.1.18/debian/patches/fix_remove_influxdb.patch 2021-02-19 10:25:44.000000000 +0000 +++ biomaj3-3.1.22/debian/patches/fix_remove_influxdb.patch 2022-06-30 14:48:27.000000000 +0000 @@ -1,4 +1,5 @@ Subject: remove influxdb feature + --> This patch is deactivated since bug #950063 was done Description: python-influxdb has a bug 950063 This patch remove influxdb optional feature waiting for influx bug to be solved diff -Nru biomaj3-3.1.18/debian/patches/series biomaj3-3.1.22/debian/patches/series --- biomaj3-3.1.18/debian/patches/series 2021-02-19 10:25:44.000000000 +0000 +++ biomaj3-3.1.22/debian/patches/series 2022-06-30 14:48:27.000000000 +0000 @@ -1,2 +1,2 @@ -fix_remove_influxdb.patch +# fix_remove_influxdb.patch fix_remove_drmaa.patch diff -Nru biomaj3-3.1.18/debian/tests/pkg-python/import-name biomaj3-3.1.22/debian/tests/pkg-python/import-name --- biomaj3-3.1.18/debian/tests/pkg-python/import-name 1970-01-01 00:00:00.000000000 +0000 +++ biomaj3-3.1.22/debian/tests/pkg-python/import-name 2022-06-30 14:48:27.000000000 +0000 @@ -0,0 +1 @@ +biomaj diff -Nru biomaj3-3.1.18/README.md biomaj3-3.1.22/README.md --- biomaj3-3.1.18/README.md 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/README.md 2022-05-04 06:55:44.000000000 +0000 @@ -77,6 +77,7 @@ * Remote access: * Optional FTP server providing authenticated or anonymous data access + * HTTP access to bank files (/db endpoint, microservice setup only) Dependencies ============ diff -Nru biomaj3-3.1.18/requirements.txt biomaj3-3.1.22/requirements.txt --- biomaj3-3.1.18/requirements.txt 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/requirements.txt 2022-05-04 06:55:44.000000000 +0000 @@ -5,7 +5,7 @@ biomaj_cli mock nose -pymongo>=3.2 +pymongo==3.12.3 pycurl tabulate py-bcrypt @@ -17,3 +17,4 @@ influxdb Yapsy==1.12.2 Jinja2 +packaging diff -Nru biomaj3-3.1.18/setup.py biomaj3-3.1.22/setup.py --- biomaj3-3.1.18/setup.py 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/setup.py 2022-05-04 06:55:44.000000000 +0000 @@ -36,7 +36,7 @@ 'url': 'http://biomaj.genouest.org', 'download_url': 'http://biomaj.genouest.org', 'author_email': 'olivier.sallou@irisa.fr', - 'version': '3.1.18', + 'version': '3.1.22', 'classifiers': [ # How mature is this project? Common values are # 3 - Alpha @@ -62,7 +62,7 @@ 'biomaj_user', 'biomaj_download', 'biomaj_process', - 'pymongo>=3.2', + 'pymongo >=3.2, <4', 'pycurl', 'py-bcrypt', 'drmaa', @@ -72,7 +72,8 @@ 'redis', 'elasticsearch', 'influxdb', - 'Yapsy==1.12.2' + 'Yapsy==1.12.2', + 'packaging' ], 'tests_require': ['nose', 'mock'], 'test_suite': 'nose.collector', diff -Nru biomaj3-3.1.18/tests/alu_list_error.properties biomaj3-3.1.22/tests/alu_list_error.properties --- biomaj3-3.1.18/tests/alu_list_error.properties 1970-01-01 00:00:00.000000000 +0000 +++ biomaj3-3.1.22/tests/alu_list_error.properties 2022-05-04 06:55:44.000000000 +0000 @@ -0,0 +1,43 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="alu.n : alu repeat element. alu.a : translation of alu.n repeats" +db.name=alu +db.type=nucleic_protein + +offline.dir.name=offline/ncbi/blast/alu_tmp +dir.version=ncbi/blast/alu + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=ftp.ncbi.nih.gov +remote.dir=/blast/db/FASTA/foo/ + +release.file= +release.regexp= +release.file.compressed= + +remote.files=^alu.*\.gz$ + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^alu\.(a|n).* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + + + +### Deployment ### + +keep.old.version=1 diff -Nru biomaj3-3.1.18/tests/biomaj_tests.py biomaj3-3.1.22/tests/biomaj_tests.py --- biomaj3-3.1.18/tests/biomaj_tests.py 2020-06-26 05:51:00.000000000 +0000 +++ biomaj3-3.1.22/tests/biomaj_tests.py 2022-05-04 06:55:44.000000000 +0000 @@ -80,7 +80,8 @@ def __copy_test_bank_properties(self): if self.bank_properties is not None: return - self.bank_properties = ['alu', 'local', 'testhttp','directhttp'] + self.bank_properties = ['alu', 'local', 'testhttp','directhttp', + 'alu_list_error'] curdir = os.path.dirname(os.path.realpath(__file__)) for b in self.bank_properties: from_file = os.path.join(curdir, b+'.properties') @@ -302,26 +303,33 @@ class TestBiomajFunctional(unittest.TestCase): + # Banks used in tests + BANKS = ['local', 'alu_list_error'] + def setUp(self): self.utils = UtilsForTest() - curdir = os.path.dirname(os.path.realpath(__file__)) BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False) - #Delete all banks - b = Bank('local') - b.banks.remove({}) - - self.config = BiomajConfig('local') - data_dir = self.config.get('data.dir') - lock_file = os.path.join(data_dir,'local.lock') - if os.path.exists(lock_file): - os.remove(lock_file) + # Clean banks used in tests + for bank_name in self.BANKS: + # Delete all releases + b = Bank(bank_name) + b.banks.remove({}) + # Delete lock files + config = BiomajConfig(bank_name) + data_dir = config.get('data.dir') + lock_file = os.path.join(data_dir, 'local.lock') + if os.path.exists(lock_file): + os.remove(lock_file) def tearDown(self): - data_dir = self.config.get('data.dir') - lock_file = os.path.join(data_dir,'local.lock') - if os.path.exists(lock_file): - os.remove(lock_file) + # Delete lock files + for bank_name in self.BANKS: + config = BiomajConfig(bank_name) + data_dir = config.get('data.dir') + lock_file = os.path.join(data_dir,'local.lock') + if os.path.exists(lock_file): + os.remove(lock_file) self.utils.clean() def test_extract_release_from_file_name(self): @@ -471,7 +479,7 @@ # Remove file if os.path.exists(tmp_remote_file): os.remove(tmp_remote_file) - + def test_fromscratch_update(self): """ Try updating twice, at second time, bank should be updated (force with fromscratc) @@ -858,3 +866,9 @@ self.fail('not owner, should not be allowed') except Exception as e: pass + + @attr('network') + def test_bank_list_error(self): + b = Bank('alu_list_error') + res = b.update() + self.assertFalse(res)