diff -Nru qiime-2020.11.1/ci/recipe/conda_build_config.yaml qiime-2021.8.0/ci/recipe/conda_build_config.yaml --- qiime-2020.11.1/ci/recipe/conda_build_config.yaml 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/ci/recipe/conda_build_config.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -python: - - 3.6 diff -Nru qiime-2020.11.1/ci/recipe/meta.yaml qiime-2021.8.0/ci/recipe/meta.yaml --- qiime-2020.11.1/ci/recipe/meta.yaml 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/ci/recipe/meta.yaml 2021-09-18 17:10:25.000000000 +0000 @@ -19,14 +19,19 @@ run: - python {{ python }} - pyyaml - - decorator + - decorator >=4,<5 - pandas >=1 - - tzlocal + # tzlocal 3 is currently broken - once this is fixed drop pin + - tzlocal <3 - python-dateutil - bibtexparser - networkx + - dill test: + requires: + - pytest + imports: - qiime2 diff -Nru qiime-2020.11.1/debian/changelog qiime-2021.8.0/debian/changelog --- qiime-2020.11.1/debian/changelog 2020-12-12 09:46:31.000000000 +0000 +++ qiime-2021.8.0/debian/changelog 2021-10-17 17:53:37.000000000 +0000 @@ -1,3 +1,23 @@ +qiime (2021.8.0-2) unstable; urgency=medium + + * Team upload. + * Enable decorator 5.x in test suite + Closes: #996469 + + -- Andreas Tille Sun, 17 Oct 2021 19:53:37 +0200 + +qiime (2021.8.0-1) unstable; urgency=medium + + [ Étienne Mollier ] + * d/watch: update pattern for GitHub archive URLs + * d/control: updated uploader address + + [ Steffen Moeller ] + * New upstream version + * Standards-Version: 4.6.0 (routine-update) + + -- Steffen Moeller Sat, 18 Sep 2021 19:10:22 +0200 + qiime (2020.11.1-1) unstable; urgency=medium * Add myself to Uploaders diff -Nru qiime-2020.11.1/debian/control qiime-2021.8.0/debian/control --- qiime-2020.11.1/debian/control 2020-12-12 09:46:31.000000000 +0000 +++ qiime-2021.8.0/debian/control 2021-10-17 17:53:37.000000000 +0000 @@ -3,21 +3,22 @@ Uploaders: Steffen Moeller , Liubov Chuprikova , Andreas Tille , - Étienne Mollier + Étienne Mollier Section: science Priority: optional Build-Depends: debhelper-compat (= 13), dh-python, python3, python3-setuptools, - python3-nose, - python3-bibtexparser, - python3-decorator, - python3-pandas, - python3-tzlocal, - python3-yaml, - python3-networkx -Standards-Version: 4.5.1 + python3-nose , + python3-bibtexparser , + python3-decorator , + python3-dill , + python3-pandas , + python3-tzlocal , + python3-yaml , + python3-networkx +Standards-Version: 4.6.0 Vcs-Browser: https://salsa.debian.org/med-team/qiime Vcs-Git: https://salsa.debian.org/med-team/qiime.git Homepage: https://qiime2.org @@ -30,6 +31,7 @@ ${python3:Depends}, python3-bibtexparser, python3-decorator, + python3-dill, python3-pandas, python3-tzlocal, python3-yaml, diff -Nru qiime-2020.11.1/debian/patches/32-bits.patch qiime-2021.8.0/debian/patches/32-bits.patch --- qiime-2020.11.1/debian/patches/32-bits.patch 2020-12-06 15:55:13.000000000 +0000 +++ qiime-2021.8.0/debian/patches/32-bits.patch 2021-10-17 17:53:37.000000000 +0000 @@ -4,9 +4,11 @@ Last-Update: 2020-12-06 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +Index: qiime/qiime2/metadata/metadata.py +=================================================================== --- qiime.orig/qiime2/metadata/metadata.py +++ qiime/qiime2/metadata/metadata.py -@@ -1162,7 +1162,7 @@ +@@ -1170,7 +1170,7 @@ class NumericMetadataColumn(MetadataColu @classmethod def _is_supported_dtype(cls, dtype): @@ -15,9 +17,11 @@ @classmethod def _normalize_(cls, series): +Index: qiime/qiime2/metadata/tests/test_metadata_column.py +=================================================================== --- qiime.orig/qiime2/metadata/tests/test_metadata_column.py +++ qiime/qiime2/metadata/tests/test_metadata_column.py -@@ -26,7 +26,7 @@ +@@ -25,7 +25,7 @@ class DummyMetadataColumn(MetadataColumn @classmethod def _is_supported_dtype(cls, dtype): diff -Nru qiime-2020.11.1/debian/patches/enable_decorator_5.patch qiime-2021.8.0/debian/patches/enable_decorator_5.patch --- qiime-2020.11.1/debian/patches/enable_decorator_5.patch 1970-01-01 00:00:00.000000000 +0000 +++ qiime-2021.8.0/debian/patches/enable_decorator_5.patch 2021-10-17 17:53:37.000000000 +0000 @@ -0,0 +1,16 @@ +Description: Enable decorator 5.x in test suite +Bug-Debian: https://bugs.debian.org/996469 +Author: Andreas Tille +Last-Update: Sun, 17 Oct 2021 14:38:51 +0200 + +--- a/ci/recipe/meta.yaml ++++ b/ci/recipe/meta.yaml +@@ -19,7 +19,7 @@ requirements: + run: + - python {{ python }} + - pyyaml +- - decorator >=4,<5 ++ - decorator >=4 + - pandas >=1 + # tzlocal 3 is currently broken - once this is fixed drop pin + - tzlocal <3 diff -Nru qiime-2020.11.1/debian/patches/python3.8.patch qiime-2021.8.0/debian/patches/python3.8.patch --- qiime-2020.11.1/debian/patches/python3.8.patch 2020-11-30 21:42:20.000000000 +0000 +++ qiime-2021.8.0/debian/patches/python3.8.patch 2021-10-17 17:53:37.000000000 +0000 @@ -5,9 +5,11 @@ Last-Update: 2020-11-30 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +Index: qiime/qiime2/core/path.py +=================================================================== --- qiime.orig/qiime2/core/path.py +++ qiime/qiime2/core/path.py -@@ -129,6 +129,14 @@ +@@ -145,6 +145,14 @@ class InternalDirectory(_ConcretePath): # Same reasoning as truediv return _ConcretePath(path, str(self)) @@ -22,11 +24,13 @@ class ArchivePath(InternalDirectory): DEFAULT_PREFIX = 'qiime2-archive-' +Index: qiime/qiime2/core/type/parse.py +=================================================================== --- qiime.orig/qiime2/core/type/parse.py +++ qiime/qiime2/core/type/parse.py -@@ -99,6 +99,9 @@ - if node is ast.Str: - return expr.s +@@ -93,6 +93,9 @@ def _convert_literals(expr): + if node is ast.Name and expr.id == 'inf': + return float('inf') + if node is ast.Constant: + return expr.value @@ -34,18 +38,3 @@ raise ValueError("Unknown literal: %r" % node) ---- qiime.orig/qiime2/sdk/plugin_manager.py -+++ qiime/qiime2/sdk/plugin_manager.py -@@ -197,7 +197,11 @@ - the user and the semantic type. The return is a dictionary of filtered - formats keyed on their string names. - """ -- if filter is not None and filter not in GetFormatFilters: -+ try: -+ if filter is not None and filter not in GetFormatFilters: -+ raise ValueError("The format filter provided: %s is not " -+ "valid.", (filter)) -+ except TypeError: - raise ValueError("The format filter provided: %s is not " - "valid.", (filter)) - diff -Nru qiime-2020.11.1/debian/patches/python3.9.patch qiime-2021.8.0/debian/patches/python3.9.patch --- qiime-2020.11.1/debian/patches/python3.9.patch 2020-11-30 22:00:50.000000000 +0000 +++ qiime-2021.8.0/debian/patches/python3.9.patch 2021-10-17 17:53:37.000000000 +0000 @@ -5,9 +5,11 @@ Last-Update: 2020-11-30 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +Index: qiime/qiime2/core/type/parse.py +=================================================================== --- qiime.orig/qiime2/core/type/parse.py +++ qiime/qiime2/core/type/parse.py -@@ -44,7 +44,10 @@ +@@ -44,7 +44,10 @@ def _expr(expr): return _build_predicate(expr.func.id, args, kwargs) if node is ast.Subscript: diff -Nru qiime-2020.11.1/debian/patches/series qiime-2021.8.0/debian/patches/series --- qiime-2020.11.1/debian/patches/series 2020-12-01 12:30:54.000000000 +0000 +++ qiime-2021.8.0/debian/patches/series 2021-10-17 17:53:37.000000000 +0000 @@ -1,3 +1,4 @@ python3.8.patch python3.9.patch 32-bits.patch +enable_decorator_5.patch diff -Nru qiime-2020.11.1/debian/rules qiime-2021.8.0/debian/rules --- qiime-2020.11.1/debian/rules 2020-11-30 21:30:18.000000000 +0000 +++ qiime-2021.8.0/debian/rules 2021-10-17 17:53:37.000000000 +0000 @@ -27,3 +27,7 @@ #ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) # LC_ALL=C.UTF-8 dh_auto_test -- -s custom --test-args="cd {build_dir}; QIIMETEST= nosetests3" #endif + +override_dh_auto_clean: + dh_auto_clean + rm -rf qiime2.egg-info diff -Nru qiime-2020.11.1/debian/TODO qiime-2021.8.0/debian/TODO --- qiime-2020.11.1/debian/TODO 2020-11-29 10:35:25.000000000 +0000 +++ qiime-2021.8.0/debian/TODO 2021-10-17 17:53:37.000000000 +0000 @@ -5,24 +5,45 @@ ________________________ - q2cli - Click-based command line interface for QIIME 2 - - q2-cutadapt - QIIME 2 plugin to work with adapters in sequence data - - q2-types - QIIME 2 plugin defining types for microbiome analysis - - q2templates - Design template package for QIIME 2 Plugins - - q2-metadata - QIIME 2 plugin for working with and visualizing Metadata + - q2-cutadapt - QIIME 2 plugin to work with adapters in sequence data - q2-demux - QIIME 2 plugin for demultiplexing of sequence reads - q2-quality-filter - QIIME2 plugin for PHRED-based filtering and trimming - q2-feature-classifier - QIIME 2 plugin supporting taxonomic classification - q2-feature-table - QIIME 2 plugin supporting operations on feature tables + - q2-metadata - QIIME 2 plugin for working with and visualizing Metadata + - q2-quality-control (deps: q2-taxa) + - q2-sample-classifier (deps: q2-feature-table) + - q2-taxa (D3-scale-chromatic, thenBy, natural-sort JavaScript dependencies) + - q2-types - QIIME 2 plugin defining types for microbiome analysis + Plugins already in the NEW QUEUE ________________________________ +Work presumed ready for upload +------------------------------ + + - q2-composition (https://github.com/qiime2/q2-composition) + + - q2-emperor + - EMPEROR (https://github.com/biocore/emperor) + + - q2-dada2: + No excuse any more: DADA2 (R package) surfaced as r-bioc-dada2 + + - q2-fragment-insertion (only the first release) + + - q2-diversity (deps: q2-feature-table, q2-emperor) + + - q2-longitudinal (deps: q2-feature-table, q2-sample-classifier) + + - unifrac (python) + The work already started ________________________ - - q2-taxa (D3-scale-chromatic, thenBy, natural-sort JavaScript dependencies) - deblur - on salsa, waiting for update of sortmerna @@ -31,37 +52,19 @@ https://salsa.debian.org/med-team/sina (FTBFS) - - q2-emperor: - TODO: EMPEROR (https://github.com/biocore/emperor) The work NOT started yet ________________________ - - q2-composition (https://github.com/qiime2/q2-composition) - - - q2-sample-classifier (deps: q2-feature-table) - - q2-vsearch (deps: q2-feature-table) - - q2-dada2: - No excuse any more: DADA2 (R package) surfaced as r-bioc-dada2 - - q2-deblur: TODO: Deblur (https://github.com/biocore/deblur) - - q2-diversity (deps: q2-feature-table, q2-emperor) - TODO: unifrac (https://github.com/biocore/unifrac) - - q2-gneiss: TODO: gneiss (https://github.com/biocore/gneiss) - - q2-longitudinal (deps: q2-feature-table, q2-sample-classifier) - - q2-phylogeny (deps: q2-alignment) - - - q2-quality-control (deps: q2-taxa) - - - q2-fragment-insertion (only the first release) - q2lint (https://github.com/qiime2/q2lint) diff -Nru qiime-2020.11.1/debian/upstream/metadata qiime-2021.8.0/debian/upstream/metadata --- qiime-2020.11.1/debian/upstream/metadata 2020-11-29 10:40:15.000000000 +0000 +++ qiime-2021.8.0/debian/upstream/metadata 2021-10-17 17:53:37.000000000 +0000 @@ -1,6 +1,6 @@ Bug-Submit: https://github.com/qiime2/qiime2/issues/new Reference: -- Author: > + - Author: > Evan Bolyen and Jai Ram Rideout and Matthew R Dillon and Nicholas A Bokulich and Christian Abnet and Gabriel A Al-Ghalith and Harriet Alexander and Eric J Alm and Manimozhiyan Arumugam and Francesco @@ -34,24 +34,29 @@ Wang and Jonathan Warren and Kyle C Weber and Chase HD Williamson and Amy D Willis and Zhenjiang Zech Xu and Jesse R Zaneveld and Yilong Zhang and Qiyun Zhu and Rob Knight and J Gregory Caporaso - Title: > + Title: > Reproducible, interactive, scalable and extensible microbiome data science using QIIME 2 - Journal: Nature Biotechnology - Volume: 37 - Pages: 852 - 857 - Year: 2019 - DOI: 10.1038/s41587-019-0209-9 - PMID: 31341288 - URL: https://www.nature.com/articles/s41587-019-0209-9 - eprint: https://www.nature.com/articles/s41587-019-0209-9.pdf + Journal: Nature Biotechnology + Volume: 37 + Pages: 852 - 857 + Year: 2019 + DOI: 10.1038/s41587-019-0209-9 + PMID: 31341288 + URL: https://www.nature.com/articles/s41587-019-0209-9 + eprint: https://www.nature.com/articles/s41587-019-0209-9.pdf Registry: -- Name: conda:bioconda - Entry: qiime2 -- Name: OMICtools - Entry: OMICS_01521 -- Name: bio.tools - Entry: qiime + - Name: conda:bioconda + Entry: qiime2 + - Name: OMICtools + Entry: OMICS_01521 + - Name: bio.tools + Entry: qiime + - Name: guix + Entry: NA + Checked: 2021-09-18 + - Name: SciCrunch + Entry: SCR_021258 Bug-Database: https://github.com/qiime2/qiime2/issues Repository: https://github.com/qiime2/qiime2.git Repository-Browse: https://github.com/qiime2/qiime2 diff -Nru qiime-2020.11.1/debian/watch qiime-2021.8.0/debian/watch --- qiime-2020.11.1/debian/watch 2020-11-29 10:35:25.000000000 +0000 +++ qiime-2021.8.0/debian/watch 2021-10-17 17:53:37.000000000 +0000 @@ -1,3 +1,3 @@ version=4 -opts=filenamemangle=s/.*\/archive\/(\d[\d.]+@ARCHIVE_EXT@)/@PACKAGE@-$1/ \ -https://github.com/qiime2/qiime2/releases .*/archive/(\d[\d.]+)@ARCHIVE_EXT@ +opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*)\.tar\.gz%qiime2-$1.tar.gz%" \ + https://github.com/qiime2/qiime2/tags (?:.*?/)?v?(\d[\d.]*)\.tar\.gz diff -Nru qiime-2020.11.1/.github/workflows/ci.yml qiime-2021.8.0/.github/workflows/ci.yml --- qiime-2020.11.1/.github/workflows/ci.yml 1970-01-01 00:00:00.000000000 +0000 +++ qiime-2021.8.0/.github/workflows/ci.yml 2021-09-18 17:10:25.000000000 +0000 @@ -0,0 +1,55 @@ +# This file is automatically generated by busywork.qiime2.org and +# template-repos - any manual edits made to this file will be erased when +# busywork performs maintenance updates. + +name: ci + +on: + pull_request: + push: + branches: + - master + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: checkout source + uses: actions/checkout@v2 + + - name: set up python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: install dependencies + run: python -m pip install --upgrade pip + + - name: lint + run: | + pip install -q https://github.com/qiime2/q2lint/archive/master.zip + q2lint + pip install -q flake8 + flake8 + + build-and-test: + needs: lint + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: checkout source + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: set up git repo for versioneer + run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + + - uses: qiime2/action-library-packaging@alpha1 + with: + package-name: qiime2 + build-target: dev + additional-tests: QIIMETEST= py.test --pyargs qiime2 + library-token: ${{ secrets.LIBRARY_TOKEN }} diff -Nru qiime-2020.11.1/LICENSE qiime-2021.8.0/LICENSE --- qiime-2020.11.1/LICENSE 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/LICENSE 2021-09-18 17:10:25.000000000 +0000 @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2016-2020, QIIME 2 development team. +Copyright (c) 2016-2021, QIIME 2 development team. All rights reserved. Redistribution and use in source and binary forms, with or without diff -Nru qiime-2020.11.1/qiime2/core/archive/archiver.py qiime-2021.8.0/qiime2/core/archive/archiver.py --- qiime-2020.11.1/qiime2/core/archive/archiver.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/archiver.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/__init__.py qiime-2021.8.0/qiime2/core/archive/format/__init__.py --- qiime-2020.11.1/qiime2/core/archive/format/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/tests/__init__.py qiime-2021.8.0/qiime2/core/archive/format/tests/__init__.py --- qiime-2020.11.1/qiime2/core/archive/format/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/tests/test_util.py qiime-2021.8.0/qiime2/core/archive/format/tests/test_util.py --- qiime-2020.11.1/qiime2/core/archive/format/tests/test_util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/tests/test_util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/tests/test_v0.py qiime-2021.8.0/qiime2/core/archive/format/tests/test_v0.py --- qiime-2020.11.1/qiime2/core/archive/format/tests/test_v0.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/tests/test_v0.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/util.py qiime-2021.8.0/qiime2/core/archive/format/util.py --- qiime-2020.11.1/qiime2/core/archive/format/util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/v0.py qiime-2021.8.0/qiime2/core/archive/format/v0.py --- qiime-2020.11.1/qiime2/core/archive/format/v0.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/v0.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/v1.py qiime-2021.8.0/qiime2/core/archive/format/v1.py --- qiime-2020.11.1/qiime2/core/archive/format/v1.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/v1.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/v2.py qiime-2021.8.0/qiime2/core/archive/format/v2.py --- qiime-2020.11.1/qiime2/core/archive/format/v2.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/v2.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/v3.py qiime-2021.8.0/qiime2/core/archive/format/v3.py --- qiime-2020.11.1/qiime2/core/archive/format/v3.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/v3.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/v4.py qiime-2021.8.0/qiime2/core/archive/format/v4.py --- qiime-2020.11.1/qiime2/core/archive/format/v4.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/v4.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/format/v5.py qiime-2021.8.0/qiime2/core/archive/format/v5.py --- qiime-2020.11.1/qiime2/core/archive/format/v5.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/format/v5.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/__init__.py qiime-2021.8.0/qiime2/core/archive/__init__.py --- qiime-2020.11.1/qiime2/core/archive/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/provenance.py qiime-2021.8.0/qiime2/core/archive/provenance.py --- qiime-2020.11.1/qiime2/core/archive/provenance.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/provenance.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,13 +1,15 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # # The full license is in the file LICENSE, distributed with this software. # ---------------------------------------------------------------------------- +import os import time import collections +import collections.abc import pkg_resources import uuid import copy @@ -317,7 +319,13 @@ self.write_action_yaml() self.write_citations_bib() - self.path.rename(final_path) + # Certain networked filesystems will experience a race + # condition on `rename`, so fall back to copying. + try: + os.rename(self.path, final_path) + except FileExistsError: + distutils.dir_util.copy_tree(str(self.path), str(final_path)) + distutils.dir_util.remove_tree(str(self.path)) def fork(self): forked = copy.copy(self) @@ -403,7 +411,7 @@ def add_input(self, name, input): if input is None: self.inputs[name] = None - elif isinstance(input, collections.Iterable): + elif isinstance(input, collections.abc.Iterable): values = [] for artifact in input: record = self.add_ancestor(artifact) diff -Nru qiime-2020.11.1/qiime2/core/archive/tests/__init__.py qiime-2021.8.0/qiime2/core/archive/tests/__init__.py --- qiime-2020.11.1/qiime2/core/archive/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/tests/test_archiver.py qiime-2021.8.0/qiime2/core/archive/tests/test_archiver.py --- qiime-2020.11.1/qiime2/core/archive/tests/test_archiver.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/tests/test_archiver.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/tests/test_citations.py qiime-2021.8.0/qiime2/core/archive/tests/test_citations.py --- qiime-2020.11.1/qiime2/core/archive/tests/test_citations.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/tests/test_citations.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/archive/tests/test_provenance.py qiime-2021.8.0/qiime2/core/archive/tests/test_provenance.py --- qiime-2020.11.1/qiime2/core/archive/tests/test_provenance.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/archive/tests/test_provenance.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -11,7 +11,6 @@ import unittest.mock as mock import pandas as pd -import pandas.util.testing as pdt import qiime2 from qiime2.plugins import dummy_plugin @@ -36,7 +35,7 @@ new_m = qiime2.Metadata.load( str(p_dir / 'artifacts' / str(b.uuid) / 'action' / 'metadata.tsv')) - pdt.assert_frame_equal(m.to_dataframe(), new_m.to_dataframe()) + pd.testing.assert_frame_equal(m.to_dataframe(), new_m.to_dataframe()) with (p_dir / 'action' / 'metadata.tsv').open() as fh: self.assertEqual( @@ -72,7 +71,7 @@ new_m = qiime2.Metadata.load( str(p_dir / 'artifacts' / str(b.uuid) / 'action' / 'metadata.tsv')) - pdt.assert_frame_equal(m.to_dataframe(), new_m.to_dataframe()) + pd.testing.assert_frame_equal(m.to_dataframe(), new_m.to_dataframe()) # Check that provenance of originating metadata artifact exists self.assertTrue((p_dir / 'artifacts' / str(metadata_artifact_1.uuid) / @@ -115,8 +114,8 @@ new_merged_md = qiime2.Metadata.load( str(p_dir / 'artifacts' / str(b.uuid) / 'action' / 'metadata.tsv')) - pdt.assert_frame_equal(new_merged_md.to_dataframe(), - merged_md.to_dataframe()) + pd.testing.assert_frame_equal(new_merged_md.to_dataframe(), + merged_md.to_dataframe()) # Check that provenance of originating metadata artifacts exists self.assertTrue((p_dir / 'artifacts' / str(md_artifact1.uuid) / @@ -262,6 +261,22 @@ self.assertEqual(obs, exp) self.assertTrue(mocked_tzlocal.called) + def test_prov_rename(self): + viz, = dummy_plugin.actions.no_input_viz() + + viz_p_dir = viz._archiver.provenance_dir + self.assertTrue(viz_p_dir.exists()) + + @mock.patch('qiime2.core.path.ProvenancePath.rename', + side_effect=FileExistsError) + def test_prov_rename_file_exists(self, _): + viz, = dummy_plugin.actions.no_input_viz() + + viz_p_dir = viz._archiver.provenance_dir + + with (viz_p_dir / 'action' / 'action.yaml').open() as fh: + self.assertIn('output-name: visualization', fh.read()) + if __name__ == '__main__': unittest.main() diff -Nru qiime-2020.11.1/qiime2/core/cite.py qiime-2021.8.0/qiime2/core/cite.py --- qiime-2020.11.1/qiime2/core/cite.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/cite.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/exceptions.py qiime-2021.8.0/qiime2/core/exceptions.py --- qiime-2020.11.1/qiime2/core/exceptions.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/exceptions.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -9,3 +9,7 @@ class ValidationError(Exception): pass + + +class ImplementationError(Exception): + pass diff -Nru qiime-2020.11.1/qiime2/core/format.py qiime-2021.8.0/qiime2/core/format.py --- qiime-2020.11.1/qiime2/core/format.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/format.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/__init__.py qiime-2021.8.0/qiime2/core/__init__.py --- qiime-2020.11.1/qiime2/core/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/path.py qiime-2021.8.0/qiime2/core/path.py --- qiime-2020.11.1/qiime2/core/path.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/path.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -27,14 +27,30 @@ self._user_owned = True return self + def _copy_dir_or_file(self, other): + if self.is_dir(): + return distutils.dir_util.copy_tree(str(self), str(other)) + else: + return shutil.copy(str(self), str(other)) + + def _destruct(self): + if self.is_dir(): + distutils.dir_util.remove_tree(str(self)) + else: + self.unlink() + def _move_or_copy(self, other): if self._user_owned: - if self.is_dir(): - return distutils.dir_util.copy_tree(str(self), str(other)) - else: - return shutil.copy(str(self), str(other)) + return self._copy_dir_or_file(other) else: - return _ConcretePath.rename(self, other) + # Certain networked filesystems will experience a race + # condition on `rename`, so fall back to copying. + try: + return _ConcretePath.rename(self, other) + except FileExistsError: + copied = self._copy_dir_or_file(other) + self._destruct() + return copied class InPath(OwnedPath): diff -Nru qiime-2020.11.1/qiime2/core/testing/examples.py qiime-2021.8.0/qiime2/core/testing/examples.py --- qiime-2020.11.1/qiime2/core/testing/examples.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/examples.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -10,8 +10,6 @@ from qiime2 import Artifact, Metadata -from qiime2.plugin import UsageAction, UsageInputs, UsageOutputNames - from .type import IntSequence1, IntSequence2, Mapping, SingleInt @@ -58,9 +56,11 @@ use.comment('This example demonstrates basic usage.') use.action( - UsageAction(plugin_id='dummy_plugin', action_id='concatenate_ints'), - UsageInputs(ints1=ints_a, ints2=ints_b, ints3=ints_c, int1=4, int2=2), - UsageOutputNames(concatenated_ints='ints_d'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='concatenate_ints'), + use.UsageInputs(ints1=ints_a, ints2=ints_b, ints3=ints_c, int1=4, + int2=2), + use.UsageOutputNames(concatenated_ints='ints_d'), ) @@ -71,17 +71,21 @@ use.comment('This example demonstrates chained usage (pt 1).') use.action( - UsageAction(plugin_id='dummy_plugin', action_id='concatenate_ints'), - UsageInputs(ints1=ints_a, ints2=ints_b, ints3=ints_c, int1=4, int2=2), - UsageOutputNames(concatenated_ints='ints_d'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='concatenate_ints'), + use.UsageInputs(ints1=ints_a, ints2=ints_b, ints3=ints_c, int1=4, + int2=2), + use.UsageOutputNames(concatenated_ints='ints_d'), ) ints_d = use.get_result('ints_d') use.comment('This example demonstrates chained usage (pt 2).') use.action( - UsageAction(plugin_id='dummy_plugin', action_id='concatenate_ints'), - UsageInputs(ints1=ints_d, ints2=ints_b, ints3=ints_c, int1=41, int2=0), - UsageOutputNames(concatenated_ints='concatenated_ints'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='concatenate_ints'), + use.UsageInputs(ints1=ints_d, ints2=ints_b, ints3=ints_c, int1=41, + int2=0), + use.UsageOutputNames(concatenated_ints='concatenated_ints'), ) @@ -90,10 +94,12 @@ mapper = use.init_data('mapper', mapping1_factory) use.action( - UsageAction(plugin_id='dummy_plugin', action_id='typical_pipeline'), - UsageInputs(int_sequence=ints, mapping=mapper, do_extra_thing=True), - UsageOutputNames(out_map='out_map', left='left', right='right', - left_viz='left_viz', right_viz='right_viz') + use.UsageAction(plugin_id='dummy_plugin', + action_id='typical_pipeline'), + use.UsageInputs(int_sequence=ints, mapping=mapper, + do_extra_thing=True), + use.UsageOutputNames(out_map='out_map', left='left', right='right', + left_viz='left_viz', right_viz='right_viz') ) @@ -102,20 +108,24 @@ mapper1 = use.init_data('mapper1', mapping1_factory) use.action( - UsageAction(plugin_id='dummy_plugin', action_id='typical_pipeline'), - UsageInputs(int_sequence=ints1, mapping=mapper1, do_extra_thing=True), - UsageOutputNames(out_map='out_map1', left='left1', right='right1', - left_viz='left_viz1', right_viz='right_viz1') + use.UsageAction(plugin_id='dummy_plugin', + action_id='typical_pipeline'), + use.UsageInputs(int_sequence=ints1, mapping=mapper1, + do_extra_thing=True), + use.UsageOutputNames(out_map='out_map1', left='left1', right='right1', + left_viz='left_viz1', right_viz='right_viz1') ) ints2 = use.get_result('left1') mapper2 = use.get_result('out_map1') use.action( - UsageAction(plugin_id='dummy_plugin', action_id='typical_pipeline'), - UsageInputs(int_sequence=ints2, mapping=mapper2, do_extra_thing=False), - UsageOutputNames(out_map='out_map2', left='left2', right='right2', - left_viz='left_viz2', right_viz='right_viz2') + use.UsageAction(plugin_id='dummy_plugin', + action_id='typical_pipeline'), + use.UsageInputs(int_sequence=ints2, mapping=mapper2, + do_extra_thing=False), + use.UsageOutputNames(out_map='out_map2', left='left2', right='right2', + left_viz='left_viz2', right_viz='right_viz2') ) right2 = use.get_result('right2') @@ -131,15 +141,23 @@ use.comment('comment 2') +def comments_only_factory(): + def comments_only_closure(use): + use.comment('comment 1') + use.comment('comment 2') + + return comments_only_closure + + def identity_with_metadata_simple(use): ints = use.init_data('ints', ints1_factory) md = use.init_metadata('md', md1_factory) use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='identity_with_metadata'), - UsageInputs(ints=ints, metadata=md), - UsageOutputNames(out='out'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='identity_with_metadata'), + use.UsageInputs(ints=ints, metadata=md), + use.UsageOutputNames(out='out'), ) @@ -151,10 +169,10 @@ md3 = use.merge_metadata('md3', md1, md2) use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='identity_with_metadata'), - UsageInputs(ints=ints, metadata=md3), - UsageOutputNames(out='out'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='identity_with_metadata'), + use.UsageInputs(ints=ints, metadata=md3), + use.UsageOutputNames(out='out'), ) @@ -164,10 +182,10 @@ mdc = use.get_metadata_column('a', md) use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='identity_with_metadata_column'), - UsageInputs(ints=ints, metadata=mdc), - UsageOutputNames(out='out'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='identity_with_metadata_column'), + use.UsageInputs(ints=ints, metadata=mdc), + use.UsageOutputNames(out='out'), ) @@ -182,10 +200,10 @@ single_int2) use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='variadic_input_method'), - UsageInputs(ints=ints, int_set=int_set, nums={7, 8, 9}), - UsageOutputNames(output='out'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='variadic_input_method'), + use.UsageInputs(ints=ints, int_set=int_set, nums={7, 8, 9}), + use.UsageOutputNames(output='out'), ) @@ -193,31 +211,31 @@ ints_a = use.init_data('ints', ints1_factory) use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='optional_artifacts_method'), - UsageInputs(ints=ints_a, num1=1), - UsageOutputNames(output='output'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='optional_artifacts_method'), + use.UsageInputs(ints=ints_a, num1=1), + use.UsageOutputNames(output='output'), ) use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='optional_artifacts_method'), - UsageInputs(ints=ints_a, num1=1, num2=2), - UsageOutputNames(output='output'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='optional_artifacts_method'), + use.UsageInputs(ints=ints_a, num1=1, num2=2), + use.UsageOutputNames(output='output'), ) use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='optional_artifacts_method'), - UsageInputs(ints=ints_a, num1=1, num2=None), - UsageOutputNames(output='ints_b'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='optional_artifacts_method'), + use.UsageInputs(ints=ints_a, num1=1, num2=None), + use.UsageOutputNames(output='ints_b'), ) ints_b = use.get_result('ints_b') use.action( - UsageAction(plugin_id='dummy_plugin', - action_id='optional_artifacts_method'), - UsageInputs(ints=ints_a, optional1=ints_b, num1=3, num2=4), - UsageOutputNames(output='output'), + use.UsageAction(plugin_id='dummy_plugin', + action_id='optional_artifacts_method'), + use.UsageInputs(ints=ints_a, optional1=ints_b, num1=3, num2=4), + use.UsageOutputNames(output='output'), ) diff -Nru qiime-2020.11.1/qiime2/core/testing/format.py qiime-2021.8.0/qiime2/core/testing/format.py --- qiime-2020.11.1/qiime2/core/testing/format.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/format.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -148,3 +148,14 @@ EchoDirectoryFormat = model.SingleFileDirectoryFormat( 'EchoDirectoryFormat', 'echo.txt', EchoFormat) + + +class Cephalapod(TextFileFormat): + """ + Class that inherits from text file format. + Used for testing validator sorting. + """ + + +CephalapodDirectoryFormat = model.SingleFileDirectoryFormat( + 'CephalapodDirectoryFormat', 'squids.tsv', Cephalapod) diff -Nru qiime-2020.11.1/qiime2/core/testing/__init__.py qiime-2021.8.0/qiime2/core/testing/__init__.py --- qiime-2020.11.1/qiime2/core/testing/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/testing/mapped.py qiime-2021.8.0/qiime2/core/testing/mapped.py --- qiime-2020.11.1/qiime2/core/testing/mapped.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/mapped.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/testing/method.py qiime-2021.8.0/qiime2/core/testing/method.py --- qiime-2020.11.1/qiime2/core/testing/method.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/method.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/testing/pipeline.py qiime-2021.8.0/qiime2/core/testing/pipeline.py --- qiime-2020.11.1/qiime2/core/testing/pipeline.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/pipeline.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/testing/plugin.py qiime-2021.8.0/qiime2/core/testing/plugin.py --- qiime-2020.11.1/qiime2/core/testing/plugin.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/plugin.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -26,11 +26,14 @@ UnimportableFormat, UnimportableDirectoryFormat, EchoFormat, - EchoDirectoryFormat + EchoDirectoryFormat, + Cephalapod, + CephalapodDirectoryFormat, ) from .type import (IntSequence1, IntSequence2, IntSequence3, Mapping, FourInts, - SingleInt, Kennel, Dog, Cat, C1, C2, C3, Foo, Bar, Baz) + SingleInt, Kennel, Dog, Cat, C1, C2, C3, Foo, Bar, Baz, + AscIntSequence, Squid, Octopus, Cuttlefish) from .method import (concatenate_ints, split_ints, merge_mappings, identity_with_metadata, identity_with_metadata_column, identity_with_categorical_metadata_column, @@ -55,6 +58,7 @@ identity_with_metadata_merging, identity_with_metadata_column_get_mdc, variadic_input_simple, optional_inputs, + comments_only_factory, ) @@ -71,17 +75,21 @@ ) import_module('qiime2.core.testing.transformer') +import_module('qiime2.core.testing.validator') + # Register semantic types dummy_plugin.register_semantic_types(IntSequence1, IntSequence2, IntSequence3, Mapping, FourInts, Kennel, Dog, Cat, - SingleInt, C1, C2, C3, Foo, Bar, Baz) + SingleInt, C1, C2, C3, Foo, Bar, Baz, + AscIntSequence, Squid, Octopus, + Cuttlefish) # Register formats dummy_plugin.register_formats( IntSequenceFormatV2, MappingFormat, IntSequenceV2DirectoryFormat, IntSequenceMultiFileDirectoryFormat, MappingDirectoryFormat, - EchoDirectoryFormat, EchoFormat) + EchoDirectoryFormat, EchoFormat, Cephalapod, CephalapodDirectoryFormat) dummy_plugin.register_formats( FourIntsDirectoryFormat, UnimportableDirectoryFormat, UnimportableFormat, @@ -132,6 +140,14 @@ | Baz, artifact_format=EchoDirectoryFormat) +dummy_plugin.register_semantic_type_to_format( + AscIntSequence, + artifact_format=IntSequenceDirectoryFormat) + +dummy_plugin.register_semantic_type_to_format( + Squid | Octopus | Cuttlefish, + artifact_format=CephalapodDirectoryFormat) + # TODO add an optional parameter to this method when they are supported dummy_plugin.methods.register_function( function=concatenate_ints, @@ -148,12 +164,15 @@ ('concatenated_ints', IntSequence1) ], name='Concatenate integers', - description='This method concatenates integers into a single sequence in ' - 'the order they are provided.', + description='This method concatenates integers into' + ' a single sequence in the order they are provided.', citations=[citations['baerheim1994effect']], examples={'concatenate_ints_simple': concatenate_ints_simple, 'concatenate_ints_complex': concatenate_ints_complex, - 'comments_only': comments_only}, + 'comments_only': comments_only, + # execute factory to make a closure to test pickling + 'comments_only_factory': comments_only_factory(), + }, ) T = TypeMatch([IntSequence1, IntSequence2]) diff -Nru qiime-2020.11.1/qiime2/core/testing/tests/__init__.py qiime-2021.8.0/qiime2/core/testing/tests/__init__.py --- qiime-2020.11.1/qiime2/core/testing/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/testing/tests/test_mapped_actions.py qiime-2021.8.0/qiime2/core/testing/tests/test_mapped_actions.py --- qiime-2020.11.1/qiime2/core/testing/tests/test_mapped_actions.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/tests/test_mapped_actions.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -204,5 +204,30 @@ self.assertEqual(x.output.type, IntSequence2) +class TestUnionedPrimitiveDecode(ActionTester): + ACTION = 'unioned_primitives' + + def test_decode_int(self): + exp = dict(foo=1, bar=1) + + res = self.action.signature.decode_parameters(foo='1', bar='1') + + self.assertEqual(res, exp) + + def test_decode_str(self): + exp = dict(foo='auto_foo', bar='auto_bar') + + res = self.action.signature.decode_parameters(**exp) + + self.assertEqual(res, exp) + + def test_decode_mix(self): + exp = dict(foo=1, bar='auto_bar') + + res = self.action.signature.decode_parameters(foo='1', bar='auto_bar') + + self.assertEqual(res, exp) + + if __name__ == '__main__': unittest.main() diff -Nru qiime-2020.11.1/qiime2/core/testing/transformer.py qiime-2021.8.0/qiime2/core/testing/transformer.py --- qiime-2020.11.1/qiime2/core/testing/transformer.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/transformer.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/testing/type.py qiime-2021.8.0/qiime2/core/testing/type.py --- qiime-2020.11.1/qiime2/core/testing/type.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/type.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -12,6 +12,7 @@ IntSequence1 = plugin.SemanticType('IntSequence1') IntSequence2 = plugin.SemanticType('IntSequence2') IntSequence3 = plugin.SemanticType('IntSequence3') +AscIntSequence = plugin.SemanticType('AscIntSequence') Mapping = plugin.SemanticType('Mapping') FourInts = plugin.SemanticType('FourInts') SingleInt = plugin.SemanticType('SingleInt') @@ -36,7 +37,11 @@ C2.field['second'], C3.field['second'], C3.field['third'] ] +# C1[C2[C3[Foo, Bar, Baz], C1[Foo]]] ... etc Foo = plugin.SemanticType('Foo', variant_of=_variants) Bar = plugin.SemanticType('Bar', variant_of=_variants) Baz = plugin.SemanticType('Baz', variant_of=_variants) -# C1[C2[C3[Foo, Bar, Baz], C1[Foo]]] ... etc + +Squid = plugin.SemanticType('Squid') +Octopus = plugin.SemanticType('Octopus') +Cuttlefish = plugin.SemanticType('Cuttlefish') diff -Nru qiime-2020.11.1/qiime2/core/testing/util.py qiime-2021.8.0/qiime2/core/testing/util.py --- qiime-2020.11.1/qiime2/core/testing/util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/testing/validator.py qiime-2021.8.0/qiime2/core/testing/validator.py --- qiime-2020.11.1/qiime2/core/testing/validator.py 1970-01-01 00:00:00.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/validator.py 2021-09-18 17:10:25.000000000 +0000 @@ -0,0 +1,56 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2016-2021, QIIME 2 development team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file LICENSE, distributed with this software. +# ---------------------------------------------------------------------------- + +from qiime2 import Metadata +from qiime2.plugin import ValidationError +from .type import (Kennel, Dog, Cat, AscIntSequence, Squid, Octopus, + Cuttlefish) +from .format import Cephalapod +from .plugin import dummy_plugin + + +@dummy_plugin.register_validator(Kennel[Dog | Cat]) +def validator_example_null1(data: dict, level): + pass + + +@dummy_plugin.register_validator(Kennel[Dog]) +def validator_example_null2(data: Metadata, level): + pass + + +@dummy_plugin.register_validator(AscIntSequence) +def validate_ascending_seq(data: list, level): + # landmine for testing + if data == [2021, 8, 24]: + raise KeyError + + prev = float('-inf') + for number in data: + if not number > prev: + raise ValidationError("%s is not greater than %s" % (number, prev)) + + +@dummy_plugin.register_validator(Squid | Cuttlefish) +def validator_sort_middle_b(data: Cephalapod, level): + pass + + +@dummy_plugin.register_validator(Squid) +def validator_sort_last(data: Cephalapod, level): + pass + + +@dummy_plugin.register_validator(Squid | Octopus | Cuttlefish) +def validator_sort_first(data: Cephalapod, level): + pass + + +@dummy_plugin.register_validator(Squid | Octopus) +def validator_sort_middle(data: Cephalapod, level): + pass diff -Nru qiime-2020.11.1/qiime2/core/testing/visualizer.py qiime-2021.8.0/qiime2/core/testing/visualizer.py --- qiime-2020.11.1/qiime2/core/testing/visualizer.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/testing/visualizer.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/tests/__init__.py qiime-2021.8.0/qiime2/core/tests/__init__.py --- qiime-2020.11.1/qiime2/core/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/tests/test_path.py qiime-2021.8.0/qiime2/core/tests/test_path.py --- qiime-2020.11.1/qiime2/core/tests/test_path.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/tests/test_path.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -7,9 +7,70 @@ # ---------------------------------------------------------------------------- import os +import pathlib +import shutil +import tempfile import unittest -from qiime2.core.path import OutPath +from qiime2.core.path import OwnedPath, OutPath + + +class TestOwnedPath(unittest.TestCase): + def setUp(self): + self.from_dir = tempfile.mkdtemp() + (pathlib.Path(self.from_dir) / 'foo.txt').touch() + + self.to_dir = tempfile.mkdtemp() + # assume to_dir is empty for all tests + + def test_move_or_copy_owned(self): + d = OwnedPath(self.from_dir) + # ensure that we are owned + d._user_owned = True + + d._move_or_copy(self.to_dir) + + # since from_dir is owned, _move_or_copy should copy, not move + self.assertTrue(os.path.exists(os.path.join(self.from_dir, 'foo.txt'))) + self.assertTrue(os.path.exists(os.path.join(self.to_dir, 'foo.txt'))) + + shutil.rmtree(self.from_dir) + shutil.rmtree(self.to_dir) + + def test_move_or_copy_not_owned_rename(self): + d = OwnedPath(self.from_dir) + # ensure that we are not owned + d._user_owned = False + + d._move_or_copy(self.to_dir) + + # since from_dir is not owned, _move_or_copy should move, not copy + self.assertFalse(os.path.exists(os.path.join(self.from_dir, + 'foo.txt'))) + self.assertTrue(os.path.exists(os.path.join(self.to_dir, 'foo.txt'))) + + with self.assertRaises(FileNotFoundError): + shutil.rmtree(self.from_dir) + shutil.rmtree(self.to_dir) + + @unittest.mock.patch('pathlib.Path.rename', side_effect=FileExistsError) + def test_move_or_copy_not_owned_copy(self, _): + d = OwnedPath(self.from_dir) + # ensure that we are not owned + d._user_owned = False + + d._move_or_copy(self.to_dir) + + # since from_dir is not owned, but the network fs race condition crops + # up, _move_or_copy should copy, not move, but then we still ensure + # that the original path has been cleaned up + self.assertFalse(os.path.exists(os.path.join(self.from_dir, + 'foo.txt'))) + self.assertTrue(os.path.exists(os.path.join(self.to_dir, 'foo.txt'))) + + with self.assertRaises(FileNotFoundError): + shutil.rmtree(self.from_dir) + shutil.rmtree(self.to_dir) class TestOutPath(unittest.TestCase): diff -Nru qiime-2020.11.1/qiime2/core/tests/test_util.py qiime-2021.8.0/qiime2/core/tests/test_util.py --- qiime-2020.11.1/qiime2/core/tests/test_util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/tests/test_util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -14,6 +14,7 @@ import dateutil.relativedelta as relativedelta import qiime2.core.util as util +from qiime2.core.testing.type import Foo, Bar, Baz class TestFindDuplicates(unittest.TestCase): @@ -286,6 +287,36 @@ self.assertEqual(fp, 'filepath/\n/with/\\newline') self.assertEqual(chks, '939aaaae6098ebdab049b0f3abe7b68c') + def test_filepath_with_leading_backslash(self): + line = r'\d41d8cd98f00b204e9800998ecf8427e \\.qza' + fp, chks = util.from_checksum_format(line) + + self.assertEqual(chks, 'd41d8cd98f00b204e9800998ecf8427e') + self.assertEqual(fp, r'\.qza') + + def test_filepath_with_leading_backslashes(self): + line = r'\d41d8cd98f00b204e9800998ecf8427e \\\\\\.qza' + fp, chks = util.from_checksum_format(line) + + self.assertEqual(fp, r'\\\.qza') + self.assertEqual(chks, 'd41d8cd98f00b204e9800998ecf8427e') + + def test_impossible_backslash(self): + # It may be impossible to generate a single '\' in the md5sum digest, + # because each '\' is escaped (as '\\') in the digest. We'll + # test for it anyway, for full coverage. + + fp, _ = util.from_checksum_format( + r'fake_checksum \.qza' + ) + + fp2, _ = util.from_checksum_format( + r'\fake_checksum \.qza' + ) + + self.assertEqual(fp, r'\.qza') + self.assertEqual(fp2, r'\.qza') + def test_from_legacy_format(self): fp, chks = util.from_checksum_format( r'0ed29022ace300b4d96847882daaf0ef *this/means/binary/mode') @@ -314,5 +345,79 @@ '9c7753f252116473994e8bffba2c620b') +class TestSortedPoset(unittest.TestCase): + def test_already_sorted_incomparable(self): + a = [Foo, Bar, Baz] + + r = util.sorted_poset(a) + + # Incomparable elements, so as long as they + # are present, any order is valid. + self.assertEqual(len(r), 3) + self.assertIn(Foo, r) + self.assertIn(Bar, r) + self.assertIn(Baz, r) + + def test_already_sorted_all_comparable(self): + a = [Foo, Foo | Bar, Foo | Bar | Baz] + + r = util.sorted_poset(a) + + self.assertEqual(a, r) + + def test_already_sorted_all_comparable_reverse(self): + a = [Foo, Foo | Bar, Foo | Bar | Baz] + + r = util.sorted_poset(a, reverse=True) + + self.assertEqual(list(reversed(a)), r) + + def test_mixed_elements(self): + a = [Foo | Bar, Foo | Baz, Foo] + + r = util.sorted_poset(a) + + self.assertEqual(r[0], Foo) + # Order of others won't matter + + def test_mxed_elements_diamond(self): + a = [Foo | Bar, Foo, Bar | Baz | Foo, Baz | Foo] + + r = util.sorted_poset(a) + + self.assertEqual(r[0], Foo) + self.assertEqual(r[-1], Bar | Baz | Foo) + + def test_multiple_minimums(self): + a = [Foo | Bar, Foo, Bar | Baz | Foo, Bar, Baz] + + r = util.sorted_poset(a) + + idx_foo = r.index(Foo) + idx_bar = r.index(Bar) + idx_foobar = r.index(Foo | Bar) + + self.assertLess(idx_foo, idx_foobar) + self.assertLess(idx_bar, idx_foobar) + self.assertEqual(r[-1], Bar | Baz | Foo) + + def test_multiple_equivalents(self): + a = [Baz, Foo | Bar, Foo, Bar | Foo, Bar] + + r = util.sorted_poset(a) + + idx_foo = r.index(Foo) + idx_bar = r.index(Bar) + idx_barfoo = r.index(Bar | Foo) + idx_foobar = r.index(Foo | Bar) + + adjacent = -1 <= idx_barfoo - idx_foobar <= 1 + self.assertTrue(adjacent) + self.assertLess(idx_foo, idx_barfoo) + self.assertLess(idx_foo, idx_foobar) + self.assertLess(idx_bar, idx_barfoo) + self.assertLess(idx_bar, idx_foobar) + + if __name__ == '__main__': unittest.main() diff -Nru qiime-2020.11.1/qiime2/core/tests/test_validate.py qiime-2021.8.0/qiime2/core/tests/test_validate.py --- qiime-2020.11.1/qiime2/core/tests/test_validate.py 1970-01-01 00:00:00.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/tests/test_validate.py 2021-09-18 17:10:25.000000000 +0000 @@ -0,0 +1,305 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2016-2021, QIIME 2 development team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file LICENSE, distributed with this software. +# ---------------------------------------------------------------------------- + +from qiime2.core.exceptions import ValidationError, ImplementationError +import unittest +from qiime2.core.validate import ValidationObject +from qiime2.sdk import PluginManager +from qiime2.plugin.plugin import ValidatorRecord, Plugin +from qiime2.core.testing.type import (IntSequence1, AscIntSequence, + Kennel, Dog, Squid, Octopus) +from qiime2.core.testing.format import IntSequenceFormat, Cephalapod + + +class TestValidationObject(unittest.TestCase): + + def setUp(self): + self.simple_int_seq = IntSequenceFormat() + + with self.simple_int_seq.open() as fh: + fh.write('\n'.join(map(str, range(3)))) + self.simple_int_seq.validate(level='max') + + def test_initialization(self): + validator_object = ValidationObject(IntSequence1) + + self.assertEqual(validator_object.concrete_type, IntSequence1) + + def test_add_validator(self): + + def test_validator_method(data: list, level): + pass + + test_record = ValidatorRecord(validator=test_validator_method, + view=list, plugin='this_plugin', + context=IntSequence1) + + validator_object = ValidationObject(IntSequence1) + + validator_object.add_validator(test_record) + + self.assertEqual(validator_object._validators, + [test_record]) + + def test_add_validation_object(self): + first_VO = ValidationObject(IntSequence1) + second_VO = ValidationObject(IntSequence1) + + def first_validator(data: list, level): + pass + + def second_validator(data: list, level): + pass + + first_record = ValidatorRecord(validator=first_validator, + view=list, plugin='this_plugin', + context=IntSequence1) + + second_record = ValidatorRecord(validator=second_validator, + view=list, plugin='this_plugin', + context=IntSequence1) + + first_VO.add_validator(first_record) + + second_VO.add_validator(second_record) + + # Allows us to demonstrate add_validation_object sets _is_sorted to + # false + first_VO._sort_validators() + + first_VO.add_validation_object(second_VO) + + self.assertEqual(first_VO._validators, [first_record, second_record]) + self.assertFalse(first_VO._is_sorted) + + def test_catch_different_concrete_types(self): + squid_vo = ValidationObject(Squid) + octopus_vo = ValidationObject(Octopus) + + def squid_validator(data: Cephalapod, level): + pass + + def octopus_validator(data: Cephalapod, level): + pass + + squid_record = ValidatorRecord(validator=squid_validator, + view=Cephalapod, + plugin='ocean_plugin', + context=Squid) + + octopus_record = ValidatorRecord(validator=octopus_validator, + view=Cephalapod, + plugin='sea_plugin', + context=Octopus) + + squid_vo.add_validator(squid_record) + octopus_vo.add_validator(octopus_record) + + with self.assertRaisesRegex(TypeError, "Unable to add"): + squid_vo.add_validation_object(octopus_vo) + + def test_public_validators_generation(self): + + validator_object = ValidationObject(IntSequence1) + + def first_validator(data: list, level): + pass + + def second_validator(data: list, level): + pass + + first_record = ValidatorRecord(validator=first_validator, + view=list, plugin='this_plugin', + context=IntSequence1) + + second_record = ValidatorRecord(validator=second_validator, + view=list, plugin='this_plugin', + context=IntSequence1) + + validator_object.add_validator(first_record) + validator_object.add_validator(second_record) + + self.assertEqual(validator_object.validators, + [first_record, second_record]) + self.assertTrue(validator_object._is_sorted) + + def test_run_validators(self): + + validator_object = ValidationObject(IntSequence1) + + has_run = False + + def test_validator_method(data: list, level): + nonlocal has_run + has_run = True + self.assertEqual(data, [0, 1, 2]) + self.assertEqual(level, 'max') + + test_record = ValidatorRecord(validator=test_validator_method, + view=list, plugin='this_plugin', + context=IntSequence1) + + validator_object.add_validator(test_record) + + validator_object(self.simple_int_seq, level='max') + + self.assertTrue(has_run) + + def test_run_validators_validation_exception(self): + validator_object = ValidationObject(AscIntSequence) + + def test_raising_validation_exception(data: list, level): + raise ValidationError("2021-08-24") + + test_record = ValidatorRecord( + validator=test_raising_validation_exception, + view=list, plugin='this_plugin', + context=AscIntSequence) + + validator_object.add_validator(test_record) + + with self.assertRaisesRegex(ValidationError, + "2021-08-24"): + validator_object(data=[], level=None) + + def test_run_validators_unknown_exception(self): + validator_object = ValidationObject(AscIntSequence) + + def test_raising_validation_exception(data: list, level): + raise KeyError("2021-08-24") + + test_record = ValidatorRecord( + validator=test_raising_validation_exception, + view=list, plugin='this_plugin', + context=AscIntSequence) + + validator_object.add_validator(test_record) + + with self.assertRaisesRegex(ImplementationError, + "attempted to validate"): + validator_object(data=[], level=None) + + def test_validator_sorts(self): + self.pm = PluginManager() + + test_object = self.pm.validators[Squid] + + self.assertFalse(test_object._is_sorted) + + exp = ['validator_sort_first', + 'validator_sort_middle', + 'validator_sort_middle_b', + 'validator_sort_last'] + + exp2 = ['validator_sort_first', + 'validator_sort_middle_b', + 'validator_sort_middle', + 'validator_sort_last'] + + obs = [record.validator.__name__ for record in test_object.validators] + + self.assertIn(obs, [exp, exp2]) + self.assertTrue(test_object._is_sorted) + + +class TestValidatorIntegration(unittest.TestCase): + + def setUp(self): + + # setup test plugin + + self.test_plugin = Plugin(name='validator_test_plugin', + version='0.0.1', + website='test.com', + package='qiime2.core.tests', + project_name='validator_test') + + self.pm = PluginManager() + + # setup test data + self.simple_int_seq = IntSequenceFormat() + + with self.simple_int_seq.open() as fh: + fh.write('\n'.join(map(str, range(3)))) + self.simple_int_seq.validate(level='max') + + def tearDown(self): + # This is a deadman switch to ensure that the test_plugin has been + # added + self.assertIn(self.test_plugin.name, self.pm.plugins) + self.pm.forget_singleton() + + def test_validator_from_each_type_in_expression(self): + @self.test_plugin.register_validator(IntSequence1 | AscIntSequence) + def blank_validator(data: list, level): + pass + + self.pm.add_plugin(self.test_plugin) + + def test_no_transformer_available(self): + @self.test_plugin.register_validator(IntSequence1 | Kennel[Dog]) + def blank_validator(data: list, level): + pass + + with self.assertRaisesRegex( + AssertionError, + r"Kennel\[Dog\].*blank_validator.*transform.*builtins:list"): + self.pm.add_plugin(self.test_plugin) + + +class TestValidatorRegistration(unittest.TestCase): + + def setUp(self): + + self.test_plugin = Plugin(name='validator_test_plugin', + version='0.0.1', + website='test.com', + package='qiime2.core.tests', + project_name='validator_test') + + def test_catch_missing_validator_arg(self): + + run_checker = False + + with self.assertRaisesRegex(TypeError, "does not contain the" + " required arguments"): + run_checker = True + + @self.test_plugin.register_validator(IntSequence1) + def validator_missing_level(data: list): + pass + + assert run_checker + + def test_catch_extra_validator_arg(self): + + run_checker = False + + with self.assertRaisesRegex(TypeError, "does not contain the" + " required arguments"): + run_checker = True + + @self.test_plugin.register_validator(IntSequence1) + def validator_extra_arg(data: list, level, spleen): + pass + + assert run_checker + + def test_catch_no_data_annotation_in_validator(self): + run_checker = False + + with self.assertRaisesRegex(TypeError, "No expected view type" + " provided as annotation for `data`" + " variable"): + run_checker = True + + @self.test_plugin.register_validator(IntSequence1) + def validator_no_view_annotation(data, level): + pass + + assert run_checker diff -Nru qiime-2020.11.1/qiime2/core/transform.py qiime-2021.8.0/qiime2/core/transform.py --- qiime-2020.11.1/qiime2/core/transform.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/transform.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -65,7 +65,7 @@ def transformation(view, validate_level='min'): view = self.coerce_view(view) - self.validate(view, validate_level) + self.validate(view, level=validate_level) new_view = transformer(view) diff -Nru qiime-2020.11.1/qiime2/core/type/collection.py qiime-2021.8.0/qiime2/core/type/collection.py --- qiime-2020.11.1/qiime2/core/type/collection.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/collection.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/grammar.py qiime-2021.8.0/qiime2/core/type/grammar.py --- qiime-2020.11.1/qiime2/core/type/grammar.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/grammar.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -592,7 +592,7 @@ if groups: elements = [] - for canidate, group in groups.items(): + for candidate, group in groups.items(): if len(group) == 1: elements.append(group[0]) else: diff -Nru qiime-2020.11.1/qiime2/core/type/__init__.py qiime-2021.8.0/qiime2/core/type/__init__.py --- qiime-2020.11.1/qiime2/core/type/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/meta.py qiime-2021.8.0/qiime2/core/type/meta.py --- qiime-2020.11.1/qiime2/core/type/meta.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/meta.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/parse.py qiime-2021.8.0/qiime2/core/type/parse.py --- qiime-2020.11.1/qiime2/core/type/parse.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/parse.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -87,18 +87,12 @@ return {_convert_literals(k): _convert_literals(v) for k, v in zip(expr.keys, expr.values)} - if node is ast.NameConstant: + if node is ast.Constant: return expr.value if node is ast.Name and expr.id == 'inf': return float('inf') - if node is ast.Num: - return expr.n - - if node is ast.Str: - return expr.s - raise ValueError("Unknown literal: %r" % node) diff -Nru qiime-2020.11.1/qiime2/core/type/primitive.py qiime-2021.8.0/qiime2/core/type/primitive.py --- qiime-2020.11.1/qiime2/core/type/primitive.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/primitive.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/semantic.py qiime-2021.8.0/qiime2/core/type/semantic.py --- qiime-2020.11.1/qiime2/core/type/semantic.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/semantic.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -7,7 +7,7 @@ # ---------------------------------------------------------------------------- import types -import collections +import collections.abc import itertools from qiime2.core.type.grammar import IncompleteExp, UnionExp, IntersectionExp @@ -116,7 +116,7 @@ if field_members is None: return fixed - if not isinstance(field_members, collections.Mapping): + if not isinstance(field_members, collections.abc.Mapping): raise ValueError("") fixed.update(field_members) diff -Nru qiime-2020.11.1/qiime2/core/type/signature.py qiime-2021.8.0/qiime2/core/type/signature.py --- qiime-2020.11.1/qiime2/core/type/signature.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/signature.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -18,7 +18,7 @@ from .primitive import infer_primitive_type from .visualization import Visualization from . import meta -from .util import is_semantic_type, is_primitive_type +from .util import is_semantic_type, is_primitive_type, parse_primitive from ..util import ImmutableBase @@ -317,7 +317,7 @@ kwargs[key] is None): params[key] = None else: - params[key] = spec.qiime_type.decode(kwargs[key]) + params[key] = parse_primitive(spec.qiime_type, kwargs[key]) return params def check_types(self, **kwargs): diff -Nru qiime-2020.11.1/qiime2/core/type/template.py qiime-2021.8.0/qiime2/core/type/template.py --- qiime-2020.11.1/qiime2/core/type/template.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/template.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/tests/__init__.py qiime-2021.8.0/qiime2/core/type/tests/__init__.py --- qiime-2020.11.1/qiime2/core/type/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/tests/test_collection.py qiime-2021.8.0/qiime2/core/type/tests/test_collection.py --- qiime-2020.11.1/qiime2/core/type/tests/test_collection.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/test_collection.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/tests/test_grammar.py qiime-2021.8.0/qiime2/core/type/tests/test_grammar.py --- qiime-2020.11.1/qiime2/core/type/tests/test_grammar.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/test_grammar.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -8,7 +8,7 @@ import pickle import unittest -import collections +import collections.abc import qiime2.core.type.grammar as grammar import qiime2.core.type.template as template @@ -291,7 +291,7 @@ Z = MockTemplate('Z') P = MockPredicate('P') - self.assertIsInstance(X, collections.Hashable) + self.assertIsInstance(X, collections.abc.Hashable) # There really shouldn't be a collision between these: self.assertNotEqual(hash(X), hash(Z % P)) @@ -626,6 +626,11 @@ repr(Foo % P | Bar % Q | Foo % R | Bar % S), 'Foo % (P | R) | Bar % (Q | S)') + self.assertEqual( + repr(grammar.UnionExp( + [Foo % P, Bar % Q, Foo % R, Bar % S]).normalize()), + 'Foo % (P | R) | Bar % (Q | S)') + def test_maximum_antichain(self): P = MockPredicate('P', alphabetize=True) Q = MockPredicate('Q', alphabetize=True) diff -Nru qiime-2020.11.1/qiime2/core/type/tests/test_meta.py qiime-2021.8.0/qiime2/core/type/tests/test_meta.py --- qiime-2020.11.1/qiime2/core/type/tests/test_meta.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/test_meta.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/tests/test_parse.py qiime-2021.8.0/qiime2/core/type/tests/test_parse.py --- qiime-2020.11.1/qiime2/core/type/tests/test_parse.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/test_parse.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/tests/test_primitive.py qiime-2021.8.0/qiime2/core/type/tests/test_primitive.py --- qiime-2020.11.1/qiime2/core/type/tests/test_primitive.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/test_primitive.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/tests/test_semantic.py qiime-2021.8.0/qiime2/core/type/tests/test_semantic.py --- qiime-2020.11.1/qiime2/core/type/tests/test_semantic.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/test_semantic.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/tests/test_util.py qiime-2021.8.0/qiime2/core/type/tests/test_util.py --- qiime-2020.11.1/qiime2/core/type/tests/test_util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/tests/test_util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/util.py qiime-2021.8.0/qiime2/core/type/util.py --- qiime-2020.11.1/qiime2/core/type/util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/type/visualization.py qiime-2021.8.0/qiime2/core/type/visualization.py --- qiime-2020.11.1/qiime2/core/type/visualization.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/type/visualization.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/core/util.py qiime-2021.8.0/qiime2/core/util.py --- qiime-2020.11.1/qiime2/core/util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -235,3 +235,25 @@ if hasattr(self, '_frozen'): _immutable_error(self) super().__setattr__(*args) + + +def sorted_poset(iterable, *, key=None, reverse=False): + values = list(iterable) + elements = values + if key is not None: + elements = [key(x) for x in values] + + result = [] + sorted_elements = [] + for value, element in zip(values, elements): + idx = 0 + for idx, placed in enumerate(sorted_elements, 1): + if element <= placed: + idx -= 1 + break + + result.insert(idx, value) + sorted_elements.insert(idx, element) + if reverse: + result = list(reversed(result)) + return result diff -Nru qiime-2020.11.1/qiime2/core/validate.py qiime-2021.8.0/qiime2/core/validate.py --- qiime-2020.11.1/qiime2/core/validate.py 1970-01-01 00:00:00.000000000 +0000 +++ qiime-2021.8.0/qiime2/core/validate.py 2021-09-18 17:10:25.000000000 +0000 @@ -0,0 +1,194 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2016-2021, QIIME 2 development team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file LICENSE, distributed with this software. +# ---------------------------------------------------------------------------- + +from qiime2.core.exceptions import ValidationError, ImplementationError +from qiime2.core.transform import ModelType +from qiime2.core.util import sorted_poset + + +class ValidationObject: + r""" + Store, sort and run all semantic validators for a for a single, complete + semantic type(a `concrete type`). + + + Attributes + ---------- + concrete_type: SemanticType + The semantic type for which the validators are valid for. + + """ + def __init__(self, concrete_type): + r""" + Create a new ValidationObject to add ValidatorRecords to. + + Parameters + ---------- + concrete_type: semantic type + The single, complete semantic type that the validators are to be + associated with. + + """ + # Private Attributes + # ------------------ + # _validators: list + # A list of ValidatorRecords + + # _is_sorted: Bool + # Tracks whether or not `_validators` has been sorted or not. + + self._validators = [] + self.concrete_type = concrete_type + self._is_sorted = False + + def add_validator(self, validator_record): + r""" + Adds new validator record to plugin. + + Parameters + ---------- + validator_record: ValidatorRecord + ValidatorRecord is a collections.namedtuple found in + `qiime2/plugin/plugin.py`. + + Notes + ----- + Used by Plugin to add a `ValidatorRecord` for a new validator to a + plugin. Usually called through the `register_validator` decorator. + + """ + self._validators.append(validator_record) + self._is_sorted = False + + def add_validation_object(self, *others): + r""" + Incorporates another validation object of the same concrete type. + + Parameters + ---------- + *others: Any number of validation objects of the same concrete type. + + Notes + ----- + Used to combine validation objects from different plugins. This is + done non-heirarchically by `PluginManager` by creating a new, blank + object for each `concrete_type` that it encounters, then adds the + objects from each plugin. + + """ + for other in others: + if self.concrete_type != other.concrete_type: + raise TypeError('Unable to add ValidationObject of' + ' `concrete_type: %s to ValidationObject of' + ' `concrete_type: %s`' % (other.concrete_type, + self.concrete_type)) + + self._validators += other._validators + self._is_sorted = False + + @property + def validators(self) -> list: + r""" + Public access method for the validators stored in ValidationObject. + + Returns + ------- + list + A sorted list of validator records. + + """ + if not self._is_sorted: + self._sort_validators() + + return self._validators + + def _sort_validators(self): + r""" + Sorts validators + + Notes + ----- + A partial order sort of the validators. The runtime for this sort is + :math:`\theta(n^2)`. This is not a concern, as the number of + validators present for any particular type is expected to remain + trivially low. The validators are sorted from general to specific. + + """ + self._validators = sorted_poset( + iterable=self._validators, + key=lambda record: record.context, + reverse=True) + + self._is_sorted = True + + def __call__(self, data, level): + r""" + Validates that provided data meets the conditions of a semantic type. + + Parameters + ---------- + data: A view of the data to be validated. + + level: {'min', 'max'} + specifies the level validation occurs at. + + Notes + ----- + Use of `level` is required but the behaviour is defined in the + individual validators. + + """ + from_mt = ModelType.from_view_type(type(data)) + + for record in self.validators: + to_mt = ModelType.from_view_type(record.view) + transformation = from_mt.make_transformation(to_mt) + data = transformation(data) + try: + record.validator(data=data, level=level) + except ValidationError: + raise + except Exception as e: + raise ImplementationError("An unexpected error occured when %r" + " from %r attempted to validate %r" + % (record.validator.__name__, + record.plugin, + data)) from e + + def assert_transformation_available(self, data): + r""" + Checks that required transformations exist. + + Parameters + ---------- + data: view + view type of input data. + + Raises + ------ + AssertionError + If no transformation exists from the data view to the view + expected by a particular validator. + + Notes + ----- + Called by `qiime2.sdk.PluginManager._consistency_check` to ensure + the transformers required to run the validators are defined. + + """ + mt = ModelType.from_view_type(data) + + for record in self._validators: + mt_other = ModelType.from_view_type(record.view) + if not mt.has_transformation(mt_other): + raise AssertionError( + 'Could not validate %s using %r because there was no' + ' transformation from %r to %r' % + (self.concrete_type, record.validator.__name__, + mt._view_name, mt_other._view_name) + ) diff -Nru qiime-2020.11.1/qiime2/__init__.py qiime-2021.8.0/qiime2/__init__.py --- qiime-2020.11.1/qiime2/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/jupyter/handlers.py qiime-2021.8.0/qiime2/jupyter/handlers.py --- qiime-2020.11.1/qiime2/jupyter/handlers.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/jupyter/handlers.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/jupyter/hooks.py qiime-2021.8.0/qiime2/jupyter/hooks.py --- qiime-2020.11.1/qiime2/jupyter/hooks.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/jupyter/hooks.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/jupyter/__init__.py qiime-2021.8.0/qiime2/jupyter/__init__.py --- qiime-2020.11.1/qiime2/jupyter/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/jupyter/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/jupyter/template.py qiime-2021.8.0/qiime2/jupyter/template.py --- qiime-2020.11.1/qiime2/jupyter/template.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/jupyter/template.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/metadata/base.py qiime-2021.8.0/qiime2/metadata/base.py --- qiime-2020.11.1/qiime2/metadata/base.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/base.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/metadata/__init__.py qiime-2021.8.0/qiime2/metadata/__init__.py --- qiime-2020.11.1/qiime2/metadata/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/metadata/io.py qiime-2021.8.0/qiime2/metadata/io.py --- qiime-2020.11.1/qiime2/metadata/io.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/io.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/metadata/metadata.py qiime-2021.8.0/qiime2/metadata/metadata.py --- qiime-2020.11.1/qiime2/metadata/metadata.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/metadata.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -203,6 +203,54 @@ return df_or_series.drop(labels=ids_to_discard, axis='index', inplace=False, errors='raise') + def save(self, filepath, ext=None): + """Save a TSV metadata file. + + The TSV metadata file format is described at https://docs.qiime2.org in + the Metadata Tutorial. + + The file will always include the ``#q2:types`` directive in order to + make the file roundtrippable without relying on column type inference. + + Parameters + ---------- + filepath : str + Path to save TSV metadata file at. + + ext : str + Preferred file extension (.tsv, .txt, etc). + Will be left blank if no extension is included. + Including a period in the extension is + optional, and any additional periods delimiting + the filepath and the extension will be reduced + to a single period. + + Returns + ------- + str + Filepath and extension (if provided) that the + file was saved to. + + See Also + -------- + Metadata.load + + """ + from .io import MetadataWriter + + if ext is None: + ext = '' + else: + ext = '.' + ext.lstrip('.') + + filepath = filepath.rstrip('.') + + if not filepath.endswith(ext): + filepath += ext + + MetadataWriter(self).write(filepath) + return filepath + # Other properties such as units can be included here in the future! ColumnProperties = collections.namedtuple('ColumnProperties', ['type']) @@ -473,28 +521,6 @@ """ return not (self == other) - def save(self, filepath): - """Save a TSV metadata file. - - The TSV metadata file format is described at https://docs.qiime2.org in - the Metadata Tutorial. - - The file will always include the ``#q2:types`` directive in order to - make the file roundtrippable without relying on column type inference. - - Parameters - ---------- - filepath : str - Path to save TSV metadata file at. - - See Also - -------- - load - - """ - from .io import MetadataWriter - MetadataWriter(self).write(filepath) - def to_dataframe(self): """Create a pandas dataframe from the metadata. @@ -938,24 +964,6 @@ """ return not (self == other) - def save(self, filepath): - """Save a TSV metadata file containing this metadata column. - - The TSV metadata file format is described at https://docs.qiime2.org in - the Metadata Tutorial. - - The file will always include the ``#q2:types`` directive in order to - make the file roundtrippable without relying on column type inference. - - Parameters - ---------- - filepath : str - Path to save TSV metadata file at. - - """ - from .io import MetadataWriter - MetadataWriter(self).write(filepath) - def to_series(self): """Create a pandas series from the metadata column. diff -Nru qiime-2020.11.1/qiime2/metadata/tests/__init__.py qiime-2021.8.0/qiime2/metadata/tests/__init__.py --- qiime-2020.11.1/qiime2/metadata/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/metadata/tests/test_io.py qiime-2021.8.0/qiime2/metadata/tests/test_io.py --- qiime-2020.11.1/qiime2/metadata/tests/test_io.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/tests/test_io.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -834,6 +834,111 @@ self.assertEqual(obs, exp) + def test_save_metadata_auto_extension(self): + md = Metadata(pd.DataFrame( + {'col1': [1.0, 2.0, 3.0], + 'col2': ['a', 'b', 'c'], + 'col3': ['foo', 'bar', '42']}, + index=pd.Index(['id1', 'id2', 'id3'], name='id'))) + + # Filename & extension endswith is matching (non-default). + fp = os.path.join(self.temp_dir, 'metadatatsv') + obs_md = md.save(fp, '.tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadatatsv.tsv') + + # No period in filename; no extension included. + fp = os.path.join(self.temp_dir, 'metadata') + obs_md = md.save(fp) + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata') + + # No period in filename; no period in extension. + fp = os.path.join(self.temp_dir, 'metadata') + obs_md = md.save(fp, 'tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # No period in filename; multiple periods in extension. + fp = os.path.join(self.temp_dir, 'metadata') + obs_md = md.save(fp, '..tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # Single period in filename; no period in extension. + fp = os.path.join(self.temp_dir, 'metadata.') + obs_md = md.save(fp, 'tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # Single period in filename; single period in extension. + fp = os.path.join(self.temp_dir, 'metadata.') + obs_md = md.save(fp, '.tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # Single period in filename; multiple periods in extension. + fp = os.path.join(self.temp_dir, 'metadata.') + obs_md = md.save(fp, '..tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # Multiple periods in filename; single period in extension. + fp = os.path.join(self.temp_dir, 'metadata..') + obs_md = md.save(fp, '.tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # Multiple periods in filename; multiple periods in extension. + fp = os.path.join(self.temp_dir, 'metadata..') + obs_md = md.save(fp, '..tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # No extension in filename; no extension input. + fp = os.path.join(self.temp_dir, 'metadata') + obs_md = md.save(fp) + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata') + + # No extension in filename; extension input. + fp = os.path.join(self.temp_dir, 'metadata') + obs_md = md.save(fp, '.tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # Extension in filename; no extension input. + fp = os.path.join(self.temp_dir, 'metadata.tsv') + obs_md = md.save(fp) + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + + # Extension in filename; extension input (non-matching). + fp = os.path.join(self.temp_dir, 'metadata.tsv') + obs_md = md.save(fp, '.txt') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv.txt') + + # Extension in filename; extension input (matching). + fp = os.path.join(self.temp_dir, 'metadata.tsv') + obs_md = md.save(fp, '.tsv') + obs_filename = os.path.basename(obs_md) + + self.assertEqual(obs_filename, 'metadata.tsv') + def test_no_bom(self): md = Metadata(pd.DataFrame( {'col1': [1.0, 2.0, 3.0], diff -Nru qiime-2020.11.1/qiime2/metadata/tests/test_metadata_column.py qiime-2021.8.0/qiime2/metadata/tests/test_metadata_column.py --- qiime-2020.11.1/qiime2/metadata/tests/test_metadata_column.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/tests/test_metadata_column.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -11,7 +11,6 @@ import unittest import pandas as pd -import pandas.util.testing as pdt import numpy as np from qiime2 import Artifact @@ -46,13 +45,15 @@ with self.assertRaisesRegex(ValueError, 'DummyMetadataColumn.*at least one ID'): DummyMetadataColumn(pd.Series([], name='col', - index=pd.Index([], name='id'))) + index=pd.Index([], name='id'), + dtype=object)) def test_invalid_id_header(self): # default index name with self.assertRaisesRegex(ValueError, r'Index\.name.*None'): DummyMetadataColumn(pd.Series([1, 2, 3], name='col', - index=pd.Index(['a', 'b', 'c']))) + index=pd.Index(['a', 'b', 'c'], + dtype=object))) with self.assertRaisesRegex(ValueError, r'Index\.name.*my-id-header'): DummyMetadataColumn(pd.Series( @@ -458,7 +459,7 @@ obs = mdc.to_series() - pdt.assert_series_equal(obs, series) + pd.testing.assert_series_equal(obs, series) def test_multiple_ids(self): series = pd.Series([-1.5, np.nan, 42], name='col', @@ -467,7 +468,7 @@ obs = mdc.to_series() - pdt.assert_series_equal(obs, series) + pd.testing.assert_series_equal(obs, series) def test_id_header_preserved(self): series = pd.Series( @@ -477,7 +478,7 @@ obs = mdc.to_series() - pdt.assert_series_equal(obs, series) + pd.testing.assert_series_equal(obs, series) self.assertEqual(obs.index.name, '#OTU ID') def test_series_copy(self): @@ -487,7 +488,7 @@ obs = mdc.to_series() - pdt.assert_series_equal(obs, series) + pd.testing.assert_series_equal(obs, series) self.assertIsNot(obs, series) @@ -501,7 +502,7 @@ exp = pd.DataFrame({'col': [0.0]}, index=pd.Index(['id1'], name='id')) - pdt.assert_frame_equal(obs, exp) + pd.testing.assert_frame_equal(obs, exp) def test_multiple_ids(self): series = pd.Series([0.0, 4.2, np.nan], name='my column', @@ -513,7 +514,7 @@ exp = pd.DataFrame({'my column': [0.0, 4.2, np.nan]}, index=pd.Index(['a', 'b', 'c'], name='id')) - pdt.assert_frame_equal(obs, exp) + pd.testing.assert_frame_equal(obs, exp) def test_id_header_preserved(self): series = pd.Series([0.0, 4.2, 123], name='my column', @@ -525,7 +526,7 @@ exp = pd.DataFrame({'my column': [0.0, 4.2, 123]}, index=pd.Index(['a', 'b', 'c'], name='#Sample ID')) - pdt.assert_frame_equal(obs, exp) + pd.testing.assert_frame_equal(obs, exp) self.assertEqual(obs.index.name, '#Sample ID') @@ -826,7 +827,7 @@ self.assertEqual(mdc.name, 'my column') obs_series = mdc.to_series() - pdt.assert_series_equal(obs_series, series) + pd.testing.assert_series_equal(obs_series, series) self.assertEqual(obs_series.dtype, object) def test_numeric_strings_preserved_as_strings(self): @@ -841,7 +842,7 @@ self.assertEqual(mdc.name, 'my column') obs_series = mdc.to_series() - pdt.assert_series_equal(obs_series, series) + pd.testing.assert_series_equal(obs_series, series) self.assertEqual(obs_series.dtype, object) def test_missing_data_normalized(self): @@ -856,7 +857,7 @@ [np.nan, 'foo', np.nan, np.nan], name='col1', index=pd.Index(['a', 'b', 'c', 'd'], name='id')) - pdt.assert_series_equal(obs, exp) + pd.testing.assert_series_equal(obs, exp) self.assertEqual(obs.dtype, object) self.assertTrue(np.isnan(obs['a'])) self.assertTrue(np.isnan(obs['c'])) @@ -873,7 +874,7 @@ np.array([np.nan, np.nan, np.nan], dtype=object), name='col1', index=pd.Index(['a', 'b', 'c'], name='id')) - pdt.assert_series_equal(obs, exp) + pd.testing.assert_series_equal(obs, exp) self.assertEqual(obs.dtype, object) def test_leading_trailing_whitespace_value(self): @@ -939,7 +940,7 @@ self.assertEqual(mdc.name, 'my column') obs_series = mdc.to_series() - pdt.assert_series_equal(obs_series, series) + pd.testing.assert_series_equal(obs_series, series) self.assertEqual(obs_series.dtype, np.float64) def test_supported_dtype_int(self): @@ -959,7 +960,7 @@ [0.0, 1.0, 42.0, -2.0], name='my column', index=pd.Index(['a', 'b', 'c', 'd'], name='id')) - pdt.assert_series_equal(obs_series, exp_series) + pd.testing.assert_series_equal(obs_series, exp_series) self.assertEqual(obs_series.dtype, np.float64) def test_missing_data_normalized(self): @@ -974,7 +975,7 @@ [np.nan, 4.2, np.nan, -5.678], name='col1', index=pd.Index(['a', 'b', 'c', 'd'], name='id')) - pdt.assert_series_equal(obs, exp) + pd.testing.assert_series_equal(obs, exp) self.assertEqual(obs.dtype, np.float64) self.assertTrue(np.isnan(obs['a'])) self.assertTrue(np.isnan(obs['c'])) @@ -990,7 +991,7 @@ [np.nan, np.nan, np.nan], name='col1', index=pd.Index(['a', 'b', 'c'], name='id')) - pdt.assert_series_equal(obs, exp) + pd.testing.assert_series_equal(obs, exp) self.assertEqual(obs.dtype, np.float64) diff -Nru qiime-2020.11.1/qiime2/metadata/tests/test_metadata.py qiime-2021.8.0/qiime2/metadata/tests/test_metadata.py --- qiime-2020.11.1/qiime2/metadata/tests/test_metadata.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/metadata/tests/test_metadata.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -11,7 +11,6 @@ import warnings import pandas as pd -import pandas.util.testing as pdt import numpy as np from qiime2 import Artifact @@ -529,8 +528,8 @@ md_no_artifact = Metadata(md_from_artifact.to_dataframe()) - pdt.assert_frame_equal(md_from_artifact.to_dataframe(), - md_no_artifact.to_dataframe()) + pd.testing.assert_frame_equal(md_from_artifact.to_dataframe(), + md_no_artifact.to_dataframe()) self.assertReallyNotEqual(md_from_artifact, md_no_artifact) def test_artifact_mismatch(self): @@ -542,7 +541,7 @@ md1 = artifact1.view(Metadata) md2 = artifact2.view(Metadata) - pdt.assert_frame_equal(md1.to_dataframe(), md2.to_dataframe()) + pd.testing.assert_frame_equal(md1.to_dataframe(), md2.to_dataframe()) self.assertReallyNotEqual(md1, md2) def test_id_mismatch(self): @@ -621,7 +620,7 @@ obs = md.to_dataframe() - pdt.assert_frame_equal(obs, df) + pd.testing.assert_frame_equal(obs, df) def test_id_header_preserved(self): df = pd.DataFrame({'col1': [42, 2.5], 'col2': ['foo', 'bar']}, @@ -630,7 +629,7 @@ obs = md.to_dataframe() - pdt.assert_frame_equal(obs, df) + pd.testing.assert_frame_equal(obs, df) self.assertEqual(obs.index.name, '#SampleID') def test_dataframe_copy(self): @@ -640,7 +639,7 @@ obs = md.to_dataframe() - pdt.assert_frame_equal(obs, df) + pd.testing.assert_frame_equal(obs, df) self.assertIsNot(obs, df) def test_retains_column_order(self): @@ -655,7 +654,7 @@ obs = md.to_dataframe() - pdt.assert_frame_equal(obs, df) + pd.testing.assert_frame_equal(obs, df) self.assertEqual(obs.columns.tolist(), ['z', 'a', 'ch']) def test_missing_data(self): @@ -680,7 +679,7 @@ dtype=object))]), index=index) - pdt.assert_frame_equal(obs, exp) + pd.testing.assert_frame_equal(obs, exp) self.assertEqual(obs.dtypes.to_dict(), {'col1': np.float64, 'NA': object, 'col3': object, 'col4': object}) @@ -707,7 +706,7 @@ 'col3': [42.0, np.nan, 0.0]}, index=index) - pdt.assert_frame_equal(obs, exp) + pd.testing.assert_frame_equal(obs, exp) self.assertEqual(obs.dtypes.to_dict(), {'col1': np.float64, 'col2': np.float64, 'col3': np.float64}) diff -Nru qiime-2020.11.1/qiime2/plugin/__init__.py qiime-2021.8.0/qiime2/plugin/__init__.py --- qiime-2020.11.1/qiime2/plugin/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -14,7 +14,6 @@ MetadataColumn, Categorical, Numeric, Properties, Range, Start, End, Choices, Bool, Set, List, Visualization, TypeMap, TypeMatch) -from qiime2.sdk.usage import UsageAction, UsageInputs, UsageOutputNames __all__ = ['TextFileFormat', 'BinaryFileFormat', 'DirectoryFormat', 'Plugin', @@ -22,4 +21,4 @@ 'Metadata', 'MetadataColumn', 'Categorical', 'Numeric', 'Properties', 'Range', 'Start', 'End', 'Choices', 'Visualization', 'TypeMap', 'TypeMatch', 'ValidationError', 'Citations', - 'CitationRecord', 'UsageAction', 'UsageInputs', 'UsageOutputNames'] + 'CitationRecord'] diff -Nru qiime-2020.11.1/qiime2/plugin/model/base.py qiime-2021.8.0/qiime2/plugin/model/base.py --- qiime-2020.11.1/qiime2/plugin/model/base.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/model/base.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/model/directory_format.py qiime-2021.8.0/qiime2/plugin/model/directory_format.py --- qiime-2020.11.1/qiime2/plugin/model/directory_format.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/model/directory_format.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -7,6 +7,7 @@ # ---------------------------------------------------------------------------- import re +import sys import pathlib from qiime2.core import transform @@ -195,6 +196,7 @@ # (arguably the code is going to be broken if defined dynamically anyways, # but better to find that out later than writing in the module namespace # even if it isn't called module-level [which is must be!]) - df = globals()[name] = type(name, (SingleFileDirectoryFormatBase,), - {'file': File(pathspec, format=format)}) + df = type(name, (SingleFileDirectoryFormatBase,), + {'file': File(pathspec, format=format)}) + df.__module__ = sys._getframe(1).f_globals.get('__name__', '__main__') return df diff -Nru qiime-2020.11.1/qiime2/plugin/model/file_format.py qiime-2021.8.0/qiime2/plugin/model/file_format.py --- qiime-2020.11.1/qiime2/plugin/model/file_format.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/model/file_format.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/model/__init__.py qiime-2021.8.0/qiime2/plugin/model/__init__.py --- qiime-2020.11.1/qiime2/plugin/model/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/model/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/model/tests/__init__.py qiime-2021.8.0/qiime2/plugin/model/tests/__init__.py --- qiime-2020.11.1/qiime2/plugin/model/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/model/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/model/tests/test_file_format.py qiime-2021.8.0/qiime2/plugin/model/tests/test_file_format.py --- qiime-2020.11.1/qiime2/plugin/model/tests/test_file_format.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/model/tests/test_file_format.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/plugin.py qiime-2021.8.0/qiime2/plugin/plugin.py --- qiime-2020.11.1/qiime2/plugin/plugin.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/plugin.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -7,10 +7,12 @@ # ---------------------------------------------------------------------------- import collections +import inspect import types import qiime2.sdk import qiime2.core.type.grammar as grammar +from qiime2.core.validate import ValidationObject from qiime2.plugin.model import DirectoryFormat from qiime2.plugin.model.base import FormatBase from qiime2.core.type import is_semantic_type @@ -28,6 +30,8 @@ 'ViewRecord', ['name', 'view', 'plugin', 'citations']) TypeFormatRecord = collections.namedtuple( 'TypeFormatRecord', ['type_expression', 'format', 'plugin']) +ValidatorRecord = collections.namedtuple( + 'ValidatorRecord', ['validator', 'view', 'plugin', 'context']) class Plugin: @@ -76,6 +80,7 @@ self.type_fragments = {} self.transformers = {} self.type_formats = [] + self.validators = {} def freeze(self): pass @@ -135,6 +140,40 @@ if is_format: self.formats[name] = FormatRecord(format=view, plugin=self) + def register_validator(self, semantic_expression): + if not is_semantic_type(semantic_expression): + raise TypeError('%s is not a Semantic Type' % semantic_expression) + + def decorator(validator): + + validator_signature = inspect.getfullargspec(validator) + + if 'data' not in validator_signature.annotations: + raise TypeError('No expected view type provided as annotation' + ' for `data` variable in %r.' % + (validator.__name__)) + + if not ['data', 'level'] == validator_signature.args: + raise TypeError('The function signature: %r does not contain' + ' the required arguments and only the required' + ' arguments: %r' % ( + validator_signature.args, + ['data', 'level'])) + + for semantic_type in semantic_expression: + if semantic_type not in self.validators: + self.validators[semantic_type] = \ + ValidationObject(semantic_type) + + self.validators[semantic_type].add_validator( + ValidatorRecord( + validator=validator, + view=validator.__annotations__['data'], + plugin=self, + context=semantic_expression)) + return validator + return decorator + def register_transformer(self, _fn=None, *, citations=None): """ A transformer has the type Callable[[type], type] diff -Nru qiime-2020.11.1/qiime2/plugin/testing.py qiime-2021.8.0/qiime2/plugin/testing.py --- qiime-2020.11.1/qiime2/plugin/testing.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/testing.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/tests/__init__.py qiime-2021.8.0/qiime2/plugin/tests/__init__.py --- qiime-2020.11.1/qiime2/plugin/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/tests/test_plugin.py qiime-2021.8.0/qiime2/plugin/tests/test_plugin.py --- qiime-2020.11.1/qiime2/plugin/tests/test_plugin.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/tests/test_plugin.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -161,7 +161,8 @@ set(types), set(['IntSequence1', 'IntSequence2', 'IntSequence3', 'Mapping', 'FourInts', 'Kennel', 'Dog', 'Cat', 'SingleInt', 'C1', 'C2', - 'C3', 'Foo', 'Bar', 'Baz'])) + 'C3', 'Foo', 'Bar', 'Baz', 'AscIntSequence', 'Squid', + 'Octopus', 'Cuttlefish'])) def test_types(self): types = self.plugin.types diff -Nru qiime-2020.11.1/qiime2/plugin/tests/test_tests.py qiime-2021.8.0/qiime2/plugin/tests/test_tests.py --- qiime-2020.11.1/qiime2/plugin/tests/test_tests.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/tests/test_tests.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugin/util.py qiime-2021.8.0/qiime2/plugin/util.py --- qiime-2020.11.1/qiime2/plugin/util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugin/util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/plugins.py qiime-2021.8.0/qiime2/plugins.py --- qiime-2020.11.1/qiime2/plugins.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/plugins.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/actiongraph.py qiime-2021.8.0/qiime2/sdk/actiongraph.py --- qiime-2020.11.1/qiime2/sdk/actiongraph.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/actiongraph.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/action.py qiime-2021.8.0/qiime2/sdk/action.py --- qiime-2020.11.1/qiime2/sdk/action.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/action.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -14,6 +14,7 @@ import itertools import decorator +import dill import qiime2.sdk import qiime2.core.type as qtype @@ -144,7 +145,7 @@ return "<%s %s>" % (self.type, self.get_import_path()) def __getstate__(self): - return { + return dill.dumps({ 'callable': self._callable, 'signature': self.signature, 'plugin_id': self.plugin_id, @@ -153,10 +154,10 @@ 'citations': self.citations, 'deprecated': self.deprecated, 'examples': self.examples, - } + }) def __setstate__(self, state): - self.__init(**state) + self.__init(**dill.loads(state)) def _bind(self, context_factory): """Bind an action to a Context factory, returning a decorated function. diff -Nru qiime-2020.11.1/qiime2/sdk/context.py qiime-2021.8.0/qiime2/sdk/context.py --- qiime-2020.11.1/qiime2/sdk/context.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/context.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/__init__.py qiime-2021.8.0/qiime2/sdk/__init__.py --- qiime-2020.11.1/qiime2/sdk/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -13,7 +13,9 @@ from .results import Results from .util import parse_type, parse_format, type_from_ast from ..core.cite import Citations +from ..core.exceptions import ValidationError, ImplementationError __all__ = ['Result', 'Results', 'Artifact', 'Visualization', 'Action', 'Method', 'Visualizer', 'Pipeline', 'PluginManager', 'parse_type', - 'parse_format', 'type_from_ast', 'Context', 'Citations'] + 'parse_format', 'type_from_ast', 'Context', 'Citations', + 'ValidationError', 'ImplementationError', ] diff -Nru qiime-2020.11.1/qiime2/sdk/plugin_manager.py qiime-2021.8.0/qiime2/sdk/plugin_manager.py --- qiime-2020.11.1/qiime2/sdk/plugin_manager.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/plugin_manager.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -14,6 +14,7 @@ import qiime2.core.type from qiime2.core.format import FormatBase from qiime2.plugin.model import SingleFileDirectoryFormatBase +from qiime2.core.validate import ValidationObject from qiime2.sdk.util import parse_type from qiime2.core.type import is_semantic_type @@ -51,8 +52,12 @@ def __new__(cls, add_plugins=True): if cls.__instance is None: self = super().__new__(cls) - self._init(add_plugins=add_plugins) cls.__instance = self + try: + self._init(add_plugins=add_plugins) + except Exception: + cls.__instance = None + raise else: if add_plugins is False: raise ValueError( @@ -60,6 +65,14 @@ 'default value for `add_plugins`.') return cls.__instance + def forget_singleton(self): + """Allows later instatiation of PluginManager to produce new object + + This is done by clearing class member which saves the instance. This + will NOT invalidate or remove the object this method is called on. + """ + self.__class__.__instance = None + def _init(self, add_plugins): self.plugins = {} self.type_fragments = {} @@ -71,6 +84,7 @@ self.views = {} self.type_formats = [] self._ff_to_sfdf = {} + self.validators = {} if add_plugins: # These are all dependent loops, each requires the loop above it to @@ -80,9 +94,18 @@ package = entry_point.module_name.split('.')[0] plugin = entry_point.load() - self.add_plugin(plugin, package, project_name) + self.add_plugin(plugin, package, project_name, + consistency_check=False) - def add_plugin(self, plugin, package=None, project_name=None): + self._consistency_check() + + def _consistency_check(self): + for semantic_type, validator_obj in self.validators.items(): + validator_obj.assert_transformation_available( + self.get_directory_format(semantic_type)) + + def add_plugin(self, plugin, package=None, project_name=None, + consistency_check=True): self.plugins[plugin.name] = plugin self._plugin_by_id[plugin.id] = plugin if plugin.package is None: @@ -102,6 +125,8 @@ self._integrate_plugin(plugin) plugin.freeze() + if consistency_check is True: + return self._consistency_check() def get_plugin(self, *, id=None, name=None): if id is None and name is None: @@ -168,6 +193,14 @@ self.formats[name] = record self.type_formats.extend(plugin.type_formats) + for semantic_type, validation_object in plugin.validators.items(): + if semantic_type not in self.validators: + self.validators[semantic_type] = \ + ValidationObject(semantic_type) + + self.validators[semantic_type].add_validation_object( + validation_object) + def get_semantic_types(self): types = {} @@ -197,7 +230,7 @@ the user and the semantic type. The return is a dictionary of filtered formats keyed on their string names. """ - if filter is not None and filter not in GetFormatFilters: + if filter is not None and not isinstance(filter, GetFormatFilters): raise ValueError("The format filter provided: %s is not " "valid.", (filter)) diff -Nru qiime-2020.11.1/qiime2/sdk/result.py qiime-2021.8.0/qiime2/sdk/result.py --- qiime-2020.11.1/qiime2/sdk/result.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/result.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -143,9 +143,46 @@ def _destructor(self): return self._archiver._destructor - def save(self, filepath): - if not filepath.endswith(self.extension): - filepath += self.extension + def save(self, filepath, ext=None): + """Save to a file. + + Parameters + ---------- + filepath : str + Path to save file at. + + extension : str + Preferred file extension (.qza, .qzv, .txt, etc). + If no preferred extension input is included, + Artifact extension will default to .qza and + Visualization extension will default to .qzv. + Including a period in the extension is + optional, and any additional periods delimiting + the filepath and the extension will be reduced + to a single period. + + Returns + ------- + str + Filepath and extension (if provided) that the + file was saved to. + + See Also + -------- + load + + """ + if ext is None: + ext = self.extension + + # This accounts for edge cases in the filename extension + # and ensures that there is only a single period in the ext. + filepath = filepath.rstrip('.') + ext = '.' + ext.lstrip('.') + + if not filepath.endswith(ext): + filepath += ext + self._archiver.save(filepath) return filepath @@ -243,6 +280,7 @@ @classmethod def _from_view(cls, type, view, view_type, provenance_capture, validate_level='min'): + type_raw = type if isinstance(type, str): type = qiime2.sdk.parse_type(type) @@ -266,6 +304,10 @@ recorder=recorder) result = transformation(view, validate_level) + if type_raw in pm.validators: + validation_object = pm.validators[type] + validation_object(data=result, level=validate_level) + artifact = cls.__new__(cls) artifact._archiver = archive.Archiver.from_data( type, output_dir_fmt, diff -Nru qiime-2020.11.1/qiime2/sdk/results.py qiime-2021.8.0/qiime2/sdk/results.py --- qiime-2020.11.1/qiime2/sdk/results.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/results.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/__init__.py qiime-2021.8.0/qiime2/sdk/tests/__init__.py --- qiime-2020.11.1/qiime2/sdk/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_actiongraph.py qiime-2021.8.0/qiime2/sdk/tests/test_actiongraph.py --- qiime-2020.11.1/qiime2/sdk/tests/test_actiongraph.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_actiongraph.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_action.py qiime-2021.8.0/qiime2/sdk/tests/test_action.py --- qiime-2020.11.1/qiime2/sdk/tests/test_action.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_action.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_artifact.py qiime-2021.8.0/qiime2/sdk/tests/test_artifact.py --- qiime-2020.11.1/qiime2/sdk/tests/test_artifact.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_artifact.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_method.py qiime-2021.8.0/qiime2/sdk/tests/test_method.py --- qiime-2020.11.1/qiime2/sdk/tests/test_method.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_method.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_pipeline.py qiime-2021.8.0/qiime2/sdk/tests/test_pipeline.py --- qiime-2020.11.1/qiime2/sdk/tests/test_pipeline.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_pipeline.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_plugin_manager.py qiime-2021.8.0/qiime2/sdk/tests/test_plugin_manager.py --- qiime-2020.11.1/qiime2/sdk/tests/test_plugin_manager.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_plugin_manager.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -15,9 +15,11 @@ from qiime2.core.testing.type import (IntSequence1, IntSequence2, IntSequence3, Mapping, FourInts, Kennel, Dog, Cat, - SingleInt, C1, C2, C3, Foo, Bar, Baz) + SingleInt, C1, C2, C3, Foo, Bar, Baz, + AscIntSequence, Squid, Octopus, + Cuttlefish) -from qiime2.core.testing.format import (IntSequenceDirectoryFormat, +from qiime2.core.testing.format import (Cephalapod, IntSequenceDirectoryFormat, MappingDirectoryFormat, IntSequenceV2DirectoryFormat, IntSequenceFormatV2, @@ -26,7 +28,12 @@ IntSequenceFormat, RedundantSingleIntDirectoryFormat, EchoFormat, - EchoDirectoryFormat) + EchoDirectoryFormat, + CephalapodDirectoryFormat) + +from qiime2.core.testing.validator import (validator_example_null1, + validate_ascending_seq, + validator_example_null2) from qiime2.core.testing.util import get_dummy_plugin @@ -43,40 +50,67 @@ exp = {'dummy-plugin': self.plugin} self.assertEqual(plugins, exp) + def test_validators(self): + self.assertEqual({Kennel[Dog], Kennel[Cat], AscIntSequence, Squid, + Octopus, Cuttlefish}, + set(self.pm.validators)) + + self.assertEqual( + set([r.validator for r in + self.pm.validators[Kennel[Dog]]._validators]), + {validator_example_null1, validator_example_null2}) + + self.assertEqual( + [r.validator for r in self.pm.validators[Kennel[Cat]]._validators], + [validator_example_null1]) + + self.assertEqual( + [r.validator + for r in self.pm.validators[AscIntSequence]._validators], + [validate_ascending_seq]) + def test_type_fragments(self): types = self.pm.type_fragments exp = { - 'IntSequence1': SemanticTypeRecord(semantic_type=IntSequence1, - plugin=self.plugin), - 'IntSequence2': SemanticTypeRecord(semantic_type=IntSequence2, - plugin=self.plugin), - 'IntSequence3': SemanticTypeRecord(semantic_type=IntSequence3, - plugin=self.plugin), - 'Mapping': SemanticTypeRecord(semantic_type=Mapping, - plugin=self.plugin), - 'FourInts': SemanticTypeRecord(semantic_type=FourInts, - plugin=self.plugin), - 'Kennel': SemanticTypeRecord(semantic_type=Kennel, - plugin=self.plugin), - 'Dog': SemanticTypeRecord(semantic_type=Dog, - plugin=self.plugin), - 'Cat': SemanticTypeRecord(semantic_type=Cat, - plugin=self.plugin), - 'SingleInt': SemanticTypeRecord(semantic_type=SingleInt, - plugin=self.plugin), - 'C1': SemanticTypeRecord(semantic_type=C1, - plugin=self.plugin), - 'C2': SemanticTypeRecord(semantic_type=C2, - plugin=self.plugin), - 'C3': SemanticTypeRecord(semantic_type=C3, - plugin=self.plugin), - 'Foo': SemanticTypeRecord(semantic_type=Foo, - plugin=self.plugin), - 'Bar': SemanticTypeRecord(semantic_type=Bar, - plugin=self.plugin), - 'Baz': SemanticTypeRecord(semantic_type=Baz, - plugin=self.plugin) + 'IntSequence1': SemanticTypeRecord(semantic_type=IntSequence1, + plugin=self.plugin), + 'IntSequence2': SemanticTypeRecord(semantic_type=IntSequence2, + plugin=self.plugin), + 'IntSequence3': SemanticTypeRecord(semantic_type=IntSequence3, + plugin=self.plugin), + 'Mapping': SemanticTypeRecord(semantic_type=Mapping, + plugin=self.plugin), + 'FourInts': SemanticTypeRecord(semantic_type=FourInts, + plugin=self.plugin), + 'Kennel': SemanticTypeRecord(semantic_type=Kennel, + plugin=self.plugin), + 'Dog': SemanticTypeRecord(semantic_type=Dog, + plugin=self.plugin), + 'Cat': SemanticTypeRecord(semantic_type=Cat, + plugin=self.plugin), + 'SingleInt': SemanticTypeRecord(semantic_type=SingleInt, + plugin=self.plugin), + 'C1': SemanticTypeRecord(semantic_type=C1, + plugin=self.plugin), + 'C2': SemanticTypeRecord(semantic_type=C2, + plugin=self.plugin), + 'C3': SemanticTypeRecord(semantic_type=C3, + plugin=self.plugin), + 'Foo': SemanticTypeRecord(semantic_type=Foo, + plugin=self.plugin), + 'Bar': SemanticTypeRecord(semantic_type=Bar, + plugin=self.plugin), + 'Baz': SemanticTypeRecord(semantic_type=Baz, + plugin=self.plugin), + 'AscIntSequence': SemanticTypeRecord(semantic_type=AscIntSequence, + plugin=self.plugin), + 'Squid': SemanticTypeRecord(semantic_type=Squid, + plugin=self.plugin), + 'Octopus': SemanticTypeRecord(semantic_type=Octopus, + plugin=self.plugin), + 'Cuttlefish': SemanticTypeRecord(semantic_type=Cuttlefish, + plugin=self.plugin), } self.assertEqual(types, exp) @@ -98,7 +132,7 @@ 'Kennel[Cat]': SemanticTypeRecord(semantic_type=Kennel[Cat], plugin=self.plugin), 'SingleInt': SemanticTypeRecord(semantic_type=SingleInt, - plugin=self.plugin) + plugin=self.plugin), } self.assertLessEqual(exp.keys(), types.keys()) @@ -139,6 +173,11 @@ 'MappingDirectoryFormat': FormatRecord(format=MappingDirectoryFormat, plugin=self.plugin), + 'Cephalapod': + FormatRecord(format=Cephalapod, plugin=self.plugin), + 'CephalapodDirectoryFormat': + FormatRecord(format=CephalapodDirectoryFormat, + plugin=self.plugin), } obs = self.pm.get_formats() diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_result.py qiime-2021.8.0/qiime2/sdk/tests/test_result.py --- qiime-2020.11.1/qiime2/sdk/tests/test_result.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_result.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -157,51 +157,279 @@ def test_save_artifact_auto_extension(self): artifact = Artifact.import_data(FourInts, [0, 0, 42, 1000]) - # No extension. + # Filename & extension endswith is matching (default). + fp = os.path.join(self.test_dir.name, 'artifactqza') + obs_fp = artifact.save(fp) + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifactqza.qza') + + # Filename & extension endswith is matching (non-default). + fp = os.path.join(self.test_dir.name, 'artifacttxt') + obs_fp = artifact.save(fp, 'txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifacttxt.txt') + + # No period in filename; no period in extension. + fp = os.path.join(self.test_dir.name, 'artifact') + obs_fp = artifact.save(fp, 'txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # No period in filename; multiple periods in extension. + fp = os.path.join(self.test_dir.name, 'artifact') + obs_fp = artifact.save(fp, '..txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # Single period in filename; no period in extension. + fp = os.path.join(self.test_dir.name, 'artifact.') + obs_fp = artifact.save(fp, 'txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # Single period in filename; single period in extension. + fp = os.path.join(self.test_dir.name, 'artifact.') + obs_fp = artifact.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # Single period in filename; multiple periods in extension. + fp = os.path.join(self.test_dir.name, 'artifact.') + obs_fp = artifact.save(fp, '..txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # Multiple periods in filename; single period in extension. + fp = os.path.join(self.test_dir.name, 'artifact..') + obs_fp = artifact.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # Multiple periods in filename; multiple periods in extension. + fp = os.path.join(self.test_dir.name, 'artifact..') + obs_fp = artifact.save(fp, '..txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # No extension in filename; no extension input. fp = os.path.join(self.test_dir.name, 'artifact') obs_fp = artifact.save(fp) obs_filename = os.path.basename(obs_fp) self.assertEqual(obs_filename, 'artifact.qza') - # Wrong extension. + # No extension in filename; different extension input. + fp = os.path.join(self.test_dir.name, 'artifact') + obs_fp = artifact.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.txt') + + # No extension in filename; default extension input. + fp = os.path.join(self.test_dir.name, 'artifact') + obs_fp = artifact.save(fp, '.qza') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.qza') + + # Different extension in filename; no extension input. fp = os.path.join(self.test_dir.name, 'artifact.zip') obs_fp = artifact.save(fp) obs_filename = os.path.basename(obs_fp) self.assertEqual(obs_filename, 'artifact.zip.qza') - # Correct extension. + # Different extension in filename; + # Different extension input (non-matching). + fp = os.path.join(self.test_dir.name, 'artifact.zip') + obs_fp = artifact.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.zip.txt') + + # Different extension in filename; + # Different extension input (matching). + fp = os.path.join(self.test_dir.name, 'artifact.zip') + obs_fp = artifact.save(fp, '.zip') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.zip') + + # Different extension in filename; default extension input. + fp = os.path.join(self.test_dir.name, 'artifact.zip') + obs_fp = artifact.save(fp, '.qza') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.zip.qza') + + # Default extension in filename; no extension input. fp = os.path.join(self.test_dir.name, 'artifact.qza') obs_fp = artifact.save(fp) obs_filename = os.path.basename(obs_fp) self.assertEqual(obs_filename, 'artifact.qza') + # Default extension in filename; different extension input. + fp = os.path.join(self.test_dir.name, 'artifact.qza') + obs_fp = artifact.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.qza.txt') + + # Default extension in filename; default extension input. + fp = os.path.join(self.test_dir.name, 'artifact.qza') + obs_fp = artifact.save(fp, '.qza') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'artifact.qza') + def test_save_visualization_auto_extension(self): visualization = Visualization._from_data_dir( self.data_dir, self.make_provenance_capture()) - # No extension. + # Filename & extension endswith is matching (default). + fp = os.path.join(self.test_dir.name, 'visualizationqzv') + obs_fp = visualization.save(fp) + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualizationqzv.qzv') + + # Filename & extension endswith is matching (non-default). + fp = os.path.join(self.test_dir.name, 'visualizationtxt') + obs_fp = visualization.save(fp, 'txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualizationtxt.txt') + + # No period in filename; no period in extension. + fp = os.path.join(self.test_dir.name, 'visualization') + obs_fp = visualization.save(fp, 'txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # No period in filename; multiple periods in extension. + fp = os.path.join(self.test_dir.name, 'visualization') + obs_fp = visualization.save(fp, '..txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # Single period in filename; no period in extension. + fp = os.path.join(self.test_dir.name, 'visualization.') + obs_fp = visualization.save(fp, 'txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # Single period in filename; single period in extension. + fp = os.path.join(self.test_dir.name, 'visualization.') + obs_fp = visualization.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # Single period in filename; multiple periods in extension. + fp = os.path.join(self.test_dir.name, 'visualization.') + obs_fp = visualization.save(fp, '..txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # Multiple periods in filename; single period in extension. + fp = os.path.join(self.test_dir.name, 'visualization..') + obs_fp = visualization.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # Multiple periods in filename; multiple periods in extension. + fp = os.path.join(self.test_dir.name, 'visualization..') + obs_fp = visualization.save(fp, '..txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # No extension in filename; no extension input. fp = os.path.join(self.test_dir.name, 'visualization') obs_fp = visualization.save(fp) obs_filename = os.path.basename(obs_fp) self.assertEqual(obs_filename, 'visualization.qzv') - # Wrong extension. + # No extension in filename; different extension input. + fp = os.path.join(self.test_dir.name, 'visualization') + obs_fp = visualization.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.txt') + + # No extension in filename; default extension input. + fp = os.path.join(self.test_dir.name, 'visualization') + obs_fp = visualization.save(fp, '.qzv') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.qzv') + + # Different extension in filename; no extension input. fp = os.path.join(self.test_dir.name, 'visualization.zip') obs_fp = visualization.save(fp) obs_filename = os.path.basename(obs_fp) self.assertEqual(obs_filename, 'visualization.zip.qzv') - # Correct extension. + # Different extension in filename; + # Different extension input (non-matching). + fp = os.path.join(self.test_dir.name, 'visualization.zip') + obs_fp = visualization.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.zip.txt') + + # Different extension in filename; + # Different extension input (matching). + fp = os.path.join(self.test_dir.name, 'visualization.zip') + obs_fp = visualization.save(fp, '.zip') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.zip') + + # Different extension in filename; default extension input. + fp = os.path.join(self.test_dir.name, 'visualization.zip') + obs_fp = visualization.save(fp, '.qzv') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.zip.qzv') + + # Default extension in filename; no extension input. fp = os.path.join(self.test_dir.name, 'visualization.qzv') obs_fp = visualization.save(fp) obs_filename = os.path.basename(obs_fp) self.assertEqual(obs_filename, 'visualization.qzv') + + # Default extension in filename; different extension input. + fp = os.path.join(self.test_dir.name, 'visualization.qzv') + obs_fp = visualization.save(fp, '.txt') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.qzv.txt') + + # Default extension in filename; default extension input. + fp = os.path.join(self.test_dir.name, 'visualization.qzv') + obs_fp = visualization.save(fp, '.qzv') + obs_filename = os.path.basename(obs_fp) + + self.assertEqual(obs_filename, 'visualization.qzv') def test_import_data_single_dirfmt_to_single_dirfmt(self): temp_data_dir = os.path.join(self.test_dir.name, 'import') diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_results.py qiime-2021.8.0/qiime2/sdk/tests/test_results.py --- qiime-2020.11.1/qiime2/sdk/tests/test_results.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_results.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_usage.py qiime-2021.8.0/qiime2/sdk/tests/test_usage.py --- qiime-2020.11.1/qiime2/sdk/tests/test_usage.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_usage.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -31,163 +31,169 @@ action = self.plugin.actions['concatenate_ints'] use = usage.DiagnosticUsage() action.examples['concatenate_ints_simple'](use) + records = use._get_records() - self.assertEqual(5, len(use.recorder)) + self.assertEqual(5, len(records)) - obs1, obs2, obs3, obs4, obs5 = use.recorder + obs1, obs2, obs3, obs4, obs5 = records.values() - self.assertEqual('init_data', obs1['source'], - use._get_record(obs1['ref']).source) - self.assertEqual('init_data', obs2['source'], - use._get_record(obs2['ref']).source) - self.assertEqual('init_data', obs3['source'], - use._get_record(obs3['ref']).source) - self.assertEqual('comment', obs4['source']) - self.assertEqual('action', obs5['source']) - - self.assertTrue('basic usage' in obs4['text']) - - self.assertEqual('dummy_plugin', obs5['action'].plugin_id) - self.assertEqual('concatenate_ints', obs5['action'].action_id) - self.assertEqual({'int1': 4, 'int2': 2, 'ints1': 'ints_a', - 'ints2': 'ints_b', 'ints3': 'ints_c'}, - obs5['input_opts']) - self.assertEqual({'concatenated_ints': 'ints_d'}, obs5['output_opts']) + self.assertEqual('init_data', obs1.source) + self.assertEqual('init_data', obs2.source) + self.assertEqual('init_data', obs3.source) + self.assertEqual('comment', obs4.source) + self.assertEqual('action', obs5.source) + + self.assertTrue('basic usage' in obs4.result['text']) + + self.assertEqual('dummy_plugin', obs5.result['plugin_id']) + self.assertEqual('concatenate_ints', obs5.result['action_id']) + self.assertEqual({'int1': 4, 'int2': 2, + 'ints1': {'ref': 'ints_a', 'source': 'init_data'}, + 'ints2': {'ref': 'ints_b', 'source': 'init_data'}, + 'ints3': {'ref': 'ints_c', 'source': 'init_data'}}, + obs5.result['input_opts']) + self.assertEqual({'concatenated_ints': 'ints_d'}, + obs5.result['output_opts']) def test_chained(self): action = self.plugin.actions['concatenate_ints'] use = usage.DiagnosticUsage() action.examples['concatenate_ints_complex'](use) + records = use._get_records() - self.assertEqual(7, len(use.recorder)) + self.assertEqual(7, len(records)) - obs1, obs2, obs3, obs4, obs5, obs6, obs7 = use.recorder + obs1, obs2, obs3, obs4, obs5, obs6, obs7 = records.values() - self.assertEqual('init_data', obs1['source'], - use._get_record(obs1['ref']).source) - self.assertEqual('init_data', obs2['source'], - use._get_record(obs2['ref']).source) - self.assertEqual('init_data', obs3['source'], - use._get_record(obs3['ref']).source) - self.assertEqual('comment', obs4['source']) - self.assertEqual('action', obs5['source']) - self.assertEqual('comment', obs6['source']) - self.assertEqual('action', obs7['source']) - - self.assertTrue('chained usage (pt 1)' in obs4['text']) - - self.assertEqual('dummy_plugin', obs5['action'].plugin_id) - self.assertEqual('concatenate_ints', obs5['action'].action_id) - self.assertEqual({'int1': 4, 'int2': 2, 'ints1': 'ints_a', - 'ints2': 'ints_b', 'ints3': 'ints_c'}, - obs5['input_opts']) - self.assertEqual({'concatenated_ints': 'ints_d'}, obs5['output_opts']) - - self.assertTrue('chained usage (pt 2)' in obs6['text']) - - self.assertEqual('dummy_plugin', obs7['action'].plugin_id) - self.assertEqual('concatenate_ints', obs7['action'].action_id) - self.assertEqual({'int1': 41, 'int2': 0, 'ints1': 'ints_d', - 'ints2': 'ints_b', 'ints3': 'ints_c'}, - obs7['input_opts']) + self.assertEqual('init_data', obs1.source) + self.assertEqual('init_data', obs2.source) + self.assertEqual('init_data', obs3.source) + self.assertEqual('comment', obs4.source) + self.assertEqual('action', obs5.source) + self.assertEqual('comment', obs6.source) + self.assertEqual('action', obs7.source) + + self.assertTrue('chained usage (pt 1)' in obs4.result['text']) + + self.assertEqual('dummy_plugin', obs5.result['plugin_id']) + self.assertEqual('concatenate_ints', obs5.result['action_id']) + self.assertEqual({'int1': 4, 'int2': 2, + 'ints1': {'ref': 'ints_a', 'source': 'init_data'}, + 'ints2': {'ref': 'ints_b', 'source': 'init_data'}, + 'ints3': {'ref': 'ints_c', 'source': 'init_data'}}, + obs5.result['input_opts']) + self.assertEqual({'concatenated_ints': 'ints_d'}, + obs5.result['output_opts']) + + self.assertTrue('chained usage (pt 2)' in obs6.result['text']) + + self.assertEqual('dummy_plugin', obs7.result['plugin_id']) + self.assertEqual('concatenate_ints', obs7.result['action_id']) + exp7 = {'int1': 41, 'int2': 0, + 'ints1': {'action_id': 'concatenate_ints', + 'input_opts': {'int1': 4, 'int2': 2, + 'ints1': {'ref': 'ints_a', + 'source': 'init_data'}, + 'ints2': {'ref': 'ints_b', + 'source': 'init_data'}, + 'ints3': {'ref': 'ints_c', + 'source': 'init_data'}}, + 'output_opt': 'concatenated_ints', + 'output_opts': {'concatenated_ints': 'ints_d'}, + 'plugin_id': 'dummy_plugin', + 'source': 'action'}, + 'ints2': {'ref': 'ints_b', 'source': 'init_data'}, + 'ints3': {'ref': 'ints_c', 'source': 'init_data'}} + self.assertEqual(exp7, obs7.result['input_opts']) self.assertEqual({'concatenated_ints': 'concatenated_ints'}, - obs7['output_opts']) + obs7.result['output_opts']) def test_comments_only(self): action = self.plugin.actions['concatenate_ints'] use = usage.DiagnosticUsage() action.examples['comments_only'](use) + records = use._get_records() - self.assertEqual(2, len(use.recorder)) + self.assertEqual(2, len(records)) - obs1, obs2 = use.recorder + obs1, obs2 = records.values() - self.assertEqual('comment', obs1['source']) - self.assertEqual('comment', obs2['source']) + self.assertEqual('comment', obs1.source) + self.assertEqual('comment', obs2.source) - self.assertEqual('comment 1', obs1['text']) - self.assertEqual('comment 2', obs2['text']) + self.assertEqual('comment 1', obs1.result['text']) + self.assertEqual('comment 2', obs2.result['text']) def test_metadata_merging(self): action = self.plugin.actions['identity_with_metadata'] use = usage.DiagnosticUsage() action.examples['identity_with_metadata_merging'](use) + records = use._get_records() - self.assertEqual(5, len(use.recorder)) + self.assertEqual(5, len(records)) - obs1, obs2, obs3, obs4, obs5 = use.recorder + obs1, obs2, obs3, obs4, obs5 = records.values() - self.assertEqual('init_data', obs1['source'], - use._get_record(obs1['ref']).source) - self.assertEqual('init_data', obs2['source'], - use._get_record(obs2['ref']).source) - self.assertEqual('init_data', obs3['source'], - use._get_record(obs3['ref']).source) - self.assertEqual('merge_metadata', obs4['source'], - use._get_record(obs4['ref']).source) - self.assertEqual('action', obs5['source']) + self.assertEqual('init_data', obs1.source) + self.assertEqual('init_metadata', obs2.source) + self.assertEqual('init_metadata', obs3.source) + self.assertEqual('merge_metadata', obs4.source) + self.assertEqual('action', obs5.source) def test_get_metadata_column(self): action = self.plugin.actions['identity_with_metadata_column'] use = usage.DiagnosticUsage() action.examples['identity_with_metadata_column_get_mdc'](use) + records = use._get_records() - self.assertEqual(4, len(use.recorder)) + self.assertEqual(4, len(records)) - obs1, obs2, obs3, obs4 = use.recorder + obs1, obs2, obs3, obs4 = records.values() - self.assertEqual('init_data', obs1['source'], - use._get_record(obs1['ref']).source) - self.assertEqual('init_data', obs2['source'], - use._get_record(obs2['ref']).source) - self.assertEqual('get_metadata_column', obs3['source'], - use._get_record(obs3['ref']).source) - self.assertEqual('action', obs4['source']) + self.assertEqual('init_data', obs1.source) + self.assertEqual('init_metadata', obs2.source) + self.assertEqual('get_metadata_column', obs3.source) + self.assertEqual('action', obs4.source) def test_use_init_collection_data(self): action = self.plugin.actions['variadic_input_method'] use = usage.DiagnosticUsage() action.examples['variadic_input_simple'](use) + records = use._get_records() - self.assertEqual(len(use.recorder), 7) + self.assertEqual(7, len(records)) - obs1, obs2, obs3, obs4, obs5, obs6, obs7 = use.recorder + obs1, obs2, obs3, obs4, obs5, obs6, obs7 = records.values() - self.assertEqual('init_data', obs1['source'], - use._get_record(obs1['ref']).source) - self.assertEqual('init_data', obs2['source'], - use._get_record(obs2['ref']).source) - self.assertEqual('init_data_collection', obs3['source'], - use._get_record(obs3['ref']).source) - - self.assertEqual('init_data', obs4['source'], - use._get_record(obs4['ref']).source) - self.assertEqual('init_data', obs5['source'], - use._get_record(obs5['ref']).source) - self.assertEqual('init_data_collection', obs6['source'], - use._get_record(obs6['ref']).source) - self.assertEqual('action', obs7['source']) + self.assertEqual('init_data', obs1.source) + self.assertEqual('init_data', obs2.source) + self.assertEqual('init_data_collection', obs3.source) - self.assertEqual(set, type(obs7['input_opts']['nums'])) + self.assertEqual('init_data', obs4.source) + self.assertEqual('init_data', obs5.source) + self.assertEqual('init_data_collection', obs6.source) + self.assertEqual('action', obs7.source) - self.assertIn('ints', obs7['input_opts']['ints']) - self.assertIn('int_set', obs7['input_opts']['int_set']) + self.assertEqual(set, type(obs7.result['input_opts']['nums'])) + + self.assertEqual('ints', obs7.result['input_opts']['ints'][0]['ref']) + self.assertEqual('int_set', + obs7.result['input_opts']['int_set'][0]['ref']) def test_optional_inputs(self): action = self.plugin.actions['optional_artifacts_method'] use = usage.DiagnosticUsage() + records = use._get_records() action.examples['optional_inputs'](use) - self.assertEqual(5, len(use.recorder)) + self.assertEqual(3, len(records)) + + obs1, obs2, obs3 = records.values() - obs1, obs2, obs3, obs4, obs5 = use.recorder - self.assertEqual('init_data', obs1['source'], - use._get_record(obs1['ref']).source) - self.assertEqual('action', obs2['source']) - self.assertEqual('action', obs3['source']) - self.assertEqual('action', obs4['source']) - self.assertEqual('action', obs5['source']) + self.assertEqual('init_data', obs1.source) + self.assertEqual('action', obs2.source) + self.assertEqual('action', obs3.source) class TestUsageAction(TestCaseUsage): @@ -409,6 +415,29 @@ action = self.plugin.actions['variadic_input_method'] action.examples['variadic_input_simple'](use) ints_a = use._get_record('ints_a') + ints_b = use._get_record('ints_b') + ints = use._get_record('ints') + single_int1 = use._get_record('single_int1') + single_int2 = use._get_record('single_int2') + int_set = use._get_record('int_set') + out = use._get_record('out') + self.assertIsInstance(ints_a.result, Artifact) + self.assertIsInstance(ints_b.result, Artifact) + self.assertIsInstance(ints.result, list) + self.assertEqual(ints.result[0], ints_a.result) + self.assertEqual(ints.result[1], ints_b.result) + self.assertIsInstance(single_int1.result, Artifact) + self.assertIsInstance(single_int2.result, Artifact) + self.assertIsInstance(int_set.result, set) + self.assertIn(single_int1.result, int_set.result) + self.assertIn(single_int2.result, int_set.result) + self.assertIsInstance(out.result, Artifact) + + def test_variadic_input_simple_async(self): + use = usage.ExecutionUsage(asynchronous=True) + action = self.plugin.actions['variadic_input_method'] + action.examples['variadic_input_simple'](use) + ints_a = use._get_record('ints_a') ints_b = use._get_record('ints_b') ints = use._get_record('ints') single_int1 = use._get_record('single_int1') diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_util.py qiime-2021.8.0/qiime2/sdk/tests/test_util.py --- qiime-2020.11.1/qiime2/sdk/tests/test_util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_visualization.py qiime-2021.8.0/qiime2/sdk/tests/test_visualization.py --- qiime-2020.11.1/qiime2/sdk/tests/test_visualization.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_visualization.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/tests/test_visualizer.py qiime-2021.8.0/qiime2/sdk/tests/test_visualizer.py --- qiime-2020.11.1/qiime2/sdk/tests/test_visualizer.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/tests/test_visualizer.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/sdk/usage.py qiime-2021.8.0/qiime2/sdk/usage.py --- qiime-2020.11.1/qiime2/sdk/usage.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/usage.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # @@ -496,6 +496,10 @@ customized for the interface or implementation. """ + UsageAction = UsageAction + UsageInputs = UsageInputs + UsageOutputNames = UsageOutputNames + def __init__(self): self._scope = Scope() @@ -637,7 +641,8 @@ raise NotImplementedError def comment(self, text: str): - return self._comment_(text) + comment = self._comment_(text) + return self._push_record(str(comment), comment, 'comment') def _comment_(self, text: str): raise NotImplementedError @@ -738,69 +743,67 @@ """ def __init__(self): super().__init__() - self.recorder = [] def _init_data_(self, ref, factory): - self.recorder.append({ + return { 'source': 'init_data', 'ref': ref, - }) - return ref + } def _init_metadata_(self, ref, factory): - self.recorder.append({ - 'source': 'init_data', + return { + 'source': 'init_metadata', 'ref': ref, - }) - return ref + } def _init_data_collection_(self, ref, collection_type, records): - self.recorder.append({ + return { 'source': 'init_data_collection', 'ref': ref, - }) - return ref, collection_type([i.ref for i in records]) + }, collection_type([i.ref for i in records]) def _merge_metadata_(self, ref, records): - self.recorder.append({ + return { 'source': 'merge_metadata', 'ref': ref, 'records_refs': [r.ref for r in records], - }) - return ref + } def _get_metadata_column_(self, column_name, record): - self.recorder.append({ + return { 'source': 'get_metadata_column', 'ref': column_name, 'record_ref': record.ref, 'column_name': column_name, - }) - return column_name + } def _comment_(self, text): - self.recorder.append({ + return { 'source': 'comment', 'text': text, - }) + } def _action_(self, action, input_opts, output_opts): - self.recorder.append({ - 'source': 'action', - 'action': action, - 'input_opts': input_opts, - 'output_opts': output_opts, - }) - return output_opts + results = dict() + for output_opt in output_opts.keys(): + results[output_opt] = { + 'source': 'action', + 'plugin_id': action.plugin_id, + 'action_id': action.action_id, + 'input_opts': input_opts, + 'output_opts': output_opts, + 'output_opt': output_opt, + } + return results def _assert_has_line_matching_(self, ref, label, path, expression): - self.recorder.append({ + return { 'source': 'assert_has_line_matching', 'ref': ref, 'label': label, 'path': path, 'expression': expression, - }) + } class ExecutionUsage(Usage): @@ -812,6 +815,10 @@ qiime2.sdk.tests.test_usage.TestExecutionUsage : Tests using this driver. qiime2.plugin.testing.TestPluginBase.execute_examples : Executes examples. """ + def __init__(self, asynchronous=False): + super().__init__() + self.asynchronous = asynchronous + def _init_data_(self, ref, factory): result = factory() result_type = type(result) @@ -860,7 +867,12 @@ input_opts: dict, output_opts: dict): action_f, _ = action.get_action() - results = action_f(**input_opts) + + if self.asynchronous: + results = action_f.asynchronous(**input_opts).result() + else: + results = action_f(**input_opts) + return {k: getattr(results, k) for k in output_opts.keys()} def _assert_has_line_matching_(self, ref, label, path, expression): diff -Nru qiime-2020.11.1/qiime2/sdk/util.py qiime-2021.8.0/qiime2/sdk/util.py --- qiime-2020.11.1/qiime2/sdk/util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/sdk/util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/tests/__init__.py qiime-2021.8.0/qiime2/tests/__init__.py --- qiime-2020.11.1/qiime2/tests/__init__.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/tests/__init__.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/tests/test_artifact_api.py qiime-2021.8.0/qiime2/tests/test_artifact_api.py --- qiime-2020.11.1/qiime2/tests/test_artifact_api.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/tests/test_artifact_api.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/tests/test_util.py qiime-2021.8.0/qiime2/tests/test_util.py --- qiime-2020.11.1/qiime2/tests/test_util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/tests/test_util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/util.py qiime-2021.8.0/qiime2/util.py --- qiime-2020.11.1/qiime2/util.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/util.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/qiime2/_version.py qiime-2021.8.0/qiime2/_version.py --- qiime-2020.11.1/qiime2/_version.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/qiime2/_version.py 2021-09-18 17:10:25.000000000 +0000 @@ -23,9 +23,9 @@ # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). - git_refnames = " (tag: 2020.11.1)" - git_full = "6502df5527ea0f7c25c0e7b9caf3ac522c8c2fda" - git_date = "2020-12-05 20:44:47 +0000" + git_refnames = " (tag: 2021.8.0)" + git_full = "a8f76ee62861bab529caa3554567b8670aa5f7b4" + git_date = "2021-09-09 18:35:32 +0000" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords diff -Nru qiime-2020.11.1/README.md qiime-2021.8.0/README.md --- qiime-2020.11.1/README.md 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/README.md 2021-09-18 17:10:25.000000000 +0000 @@ -1,26 +1,39 @@ -# QIIME 2 +# qiime2 (the QIIME 2 framework) -[![Build Status](https://travis-ci.org/qiime2/qiime2.svg?branch=master)](https://travis-ci.org/qiime2/qiime2) +![](https://github.com/qiime2/qiime2/workflows/ci/badge.svg) Source code repository for the QIIME 2 framework. -QIIME 2™ is a powerful, extensible, and decentralized microbiome bioinformatics platform that is free, open source, and community developed. With a focus on data and analysis transparency, QIIME 2 enables researchers to start an analysis with raw DNA sequence data and finish with publication-quality figures and statistical results. +QIIME 2™ is a powerful, extensible, and decentralized microbiome bioinformatics +platform that is free, open source, and community developed. With a focus on +data and analysis transparency, QIIME 2 enables researchers to start an +analysis with raw DNA sequence data and finish with publication-quality figures +and statistical results. -Visit [https://qiime2.org](https://qiime2.org) to learn more about the QIIME 2 project. +Visit [https://qiime2.org](https://qiime2.org) to learn more about the QIIME 2 +project. ## Installation -Detailed instructions are available in the [documentation](https://docs.qiime2.org/). +Detailed instructions are available in the +[documentation](https://docs.qiime2.org/). ## Users -Head to the [user docs](https://docs.qiime2.org/) for help getting started, core concepts, tutorials, and other resources. -Just have a question? Please ask it in our [forum](https://forum.qiime2.org/c/user-support). +Head to the [user docs](https://docs.qiime2.org/) for help getting started, +core concepts, tutorials, and other resources. + +Just have a question? Please ask it in our +[forum](https://forum.qiime2.org/c/user-support). ## Developers -Please visit the [contributing page](https://github.com/qiime2/qiime2/blob/master/.github/CONTRIBUTING.md) for more information on contributions, documentation links, and more. + +Please visit the [contributing page](https://dev.qiime2.org) for more +information on contributions, documentation links, and more. ## Citing QIIME 2 -If you use QIIME 2 for any published research, please include the following citation: + +If you use QIIME 2 for any published research, please include the following +citation: > Bolyen E, Rideout JR, Dillon MR, Bokulich NA, Abnet CC, Al-Ghalith GA, Alexander H, Alm EJ, Arumugam M, Asnicar F, Bai Y, Bisanz JE, Bittinger K, Brejnrod A, Brislawn CJ, Brown CT, Callahan BJ, Caraballo-Rodríguez AM, Chase J, Cope EK, Da Silva R, Diener C, Dorrestein PC, Douglas GM, Durall DM, Duvallet C, Edwardson CF, Ernst M, Estaki M, Fouquier J, Gauglitz JM, Gibbons SM, Gibson DL, Gonzalez A, Gorlick K, Guo J, Hillmann B, Holmes S, Holste H, Huttenhower C, Huttley GA, Janssen S, Jarmusch AK, Jiang L, Kaehler BD, Kang KB, Keefe CR, Keim P, Kelley ST, Knights D, Koester I, Kosciolek T, Kreps J, Langille MGI, Lee J, Ley R, Liu YX, Loftfield E, Lozupone C, Maher M, Marotz C, Martin BD, McDonald D, McIver LJ, Melnik AV, Metcalf JL, Morgan SC, Morton JT, Naimey AT, Navas-Molina JA, Nothias LF, Orchanian SB, Pearson T, Peoples SL, Petras D, Preuss ML, Pruesse E, Rasmussen LB, Rivers A, Robeson MS, Rosenthal P, Segata N, Shaffer M, Shiffer A, Sinha R, Song SJ, Spear JR, Swafford AD, Thompson LR, Torres PJ, Trinh P, Tripathi A, Turnbaugh PJ, Ul-Hasan S, van der Hooft JJJ, Vargas F, Vázquez-Baeza Y, Vogtmann E, von Hippel M, Walters W, Wan Y, Wang M, Warren J, Weber KC, Williamson CHD, Willis AD, Xu ZZ, Zaneveld JR, Zhang Y, Zhu Q, Knight R, and Caporaso JG. 2019. Reproducible, interactive, scalable and extensible microbiome data science using QIIME 2. Nature Biotechnology 37:852–857. https://doi.org/10.1038/s41587-019-0209-9 diff -Nru qiime-2020.11.1/setup.py qiime-2021.8.0/setup.py --- qiime-2020.11.1/setup.py 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/setup.py 2021-09-18 17:10:25.000000000 +0000 @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------- -# Copyright (c) 2016-2020, QIIME 2 development team. +# Copyright (c) 2016-2021, QIIME 2 development team. # # Distributed under the terms of the Modified BSD License. # diff -Nru qiime-2020.11.1/.travis.yml qiime-2021.8.0/.travis.yml --- qiime-2020.11.1/.travis.yml 2020-12-05 20:44:47.000000000 +0000 +++ qiime-2021.8.0/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,23 +0,0 @@ -dist: trusty -sudo: false -language: python -before_install: - - export MPLBACKEND='Agg' - - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - - export MINICONDA_PREFIX="$HOME/miniconda" - - bash miniconda.sh -b -p $MINICONDA_PREFIX - - export PATH="$MINICONDA_PREFIX/bin:$PATH" - - conda config --set always_yes yes - - conda update -q conda - - conda info -a -install: - - wget -q https://raw.githubusercontent.com/qiime2/environment-files/master/latest/staging/qiime2-latest-py36-linux-conda.yml - - conda env create -q -n test-env --file qiime2-latest-py36-linux-conda.yml - - source activate test-env - - pip install -q flake8 - - conda install nose - - pip install -q https://github.com/qiime2/q2lint/archive/master.zip - - make install -script: - - make lint - - make test