diff -Nru python-debian-0.1.44/debian/changelog python-debian-0.1.46/debian/changelog --- python-debian-0.1.44/debian/changelog 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/debian/changelog 2022-07-08 16:45:29.000000000 +0000 @@ -1,3 +1,28 @@ +python-debian (0.1.46) unstable; urgency=medium + + * Copyright.add_files_paragraph(): Append after the last existing + files paragraph. + * RTS parser: Fix removing and then re-adding a paragraph. + + -- Jelmer Vernooij Fri, 08 Jul 2022 17:45:29 +0100 + +python-debian (0.1.45) unstable; urgency=medium + + [ Jelmer Vernooij ] + * Add Deb822FileElement.remove method. + * RTS parser: don't add trailing whitespace when setting field values + that start with a newline. Closes: #1013485 + * RTS parser: Add Deb822FileElement.remove(). + + [ Niels Thykier ] + * RTS parser: minor performance improvements (~5%). + * RTS parser: Stop preserving lack of newlines at EOF. Closes: #998715 + * Add substvars module for handling substvars. + * copyright: Use RTS parser. + * Provide `DpkgArchTable` class as a subset of `Dpkg::Arch`. Closes: #771058 + + -- Jelmer Vernooij Tue, 05 Jul 2022 16:35:04 +0100 + python-debian (0.1.44) unstable; urgency=medium [ Simon Chopin ] diff -Nru python-debian-0.1.44/debian/copyright python-debian-0.1.46/debian/copyright --- python-debian-0.1.44/debian/copyright 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/debian/copyright 2022-07-08 16:45:29.000000000 +0000 @@ -70,6 +70,13 @@ Copyright: Copyright (C) 2021 Niels Thykier License: GPL-2+ +Files: lib/debian/_arch_table.py +Copyright: Copyright © 2006-2015 Guillem Jover + Copyright (C) 2014, Ansgar Burchardt + Copyright (C) 2014-2017, Johannes Schauer Marin Rodrigues + Copyright (C) 2022, Niels Thykier +License: GPL-2+ + License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License diff -Nru python-debian-0.1.44/docs/bts-usercategories.txt python-debian-0.1.46/docs/bts-usercategories.txt --- python-debian-0.1.44/docs/bts-usercategories.txt 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/docs/bts-usercategories.txt 2022-07-08 16:45:29.000000000 +0000 @@ -11,6 +11,7 @@ + changelog module [5:changelog] + debfile/arfile modules [5:debfile] + debtags module [5:debtags] + + substvars module [5:substvars] + watch module [5:watch] + new modules [9:new] diff -Nru python-debian-0.1.44/docs/contributing.rst python-debian-0.1.46/docs/contributing.rst --- python-debian-0.1.44/docs/contributing.rst 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/docs/contributing.rst 2022-07-08 16:45:29.000000000 +0000 @@ -50,18 +50,21 @@ Notable specifications: - `Debian Policy`_ - - `dpkg-dev man pages `_ including: - - `deb-control(5) `_, + - `dpkg-dev man pages `_ including: + - `deb-control(5) `_, the `control` file in the binary package (generated from `debian/control` in the source package) - - `deb-version(5) `_, + - `deb-version(5) `_, Debian version strings. - - `deb-changelog(5) `_, + - `deb-changelog(5) `_, changelogs for Debian packages. - - `deb-changes(5) `_, + - `deb-changes(5) `_, `changes` files that developers upload to add new packages to the archive. - - `dsc(5) `_, + - `deb-substvars(5) `_, + `substvars` files that track substitution variables in packaging that + help automate package steps. + - `dsc(5) `_, Debian Source Control file that defines the files that are part of a source package. - `Debian mirror format `_, @@ -162,7 +165,7 @@ .. include:: bts-usercategories.txt :literal: -The usertags are derived from the Python names of he (sub)modules. +The usertags are derived from the Python names of the (sub)modules. Note that usertags cannot include underscores and thus the the Python module name `debian_support` becomes the BTS usertag `debian-support`. diff -Nru python-debian-0.1.44/docs/index.rst python-debian-0.1.46/docs/index.rst --- python-debian-0.1.44/docs/index.rst 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/docs/index.rst 2022-07-08 16:45:29.000000000 +0000 @@ -27,6 +27,7 @@ api/debian.debian_support api/debian.debtags api/debian.deprecation + api/debian.substvars api/debian._deb822_repro api/debian._deb822_repro.formatter api/debian._deb822_repro.parsing diff -Nru python-debian-0.1.44/.gitlab-ci.yml python-debian-0.1.46/.gitlab-ci.yml --- python-debian-0.1.44/.gitlab-ci.yml 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/.gitlab-ci.yml 2022-07-08 16:45:29.000000000 +0000 @@ -19,13 +19,12 @@ .unit-tests: script: - ./debian/rules lib/debian/__init__.py - - LC_ALL=C py.test-3 --doctest-modules $COVERAGE --verbose $PYTEST_IGNORES lib/ - - LC_ALL=C.UTF-8 py.test-3 --doctest-modules $COVERAGE $COVERAGE_REPORT --verbose $PYTEST_IGNORES lib/ + - LC_ALL=C py.test-3 --doctest-modules $COVERAGE --verbose lib/ + - LC_ALL=C.UTF-8 py.test-3 --doctest-modules $COVERAGE $COVERAGE_REPORT --verbose lib/ variables: # Only generate coverage data and a report once in the test matrix COVERAGE: "" COVERAGE_REPORT: "" - PYTEST_IGNORES: "" unit-tests: extends: @@ -33,6 +32,7 @@ - .unit-tests after_script: - python3-coverage html + coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' variables: # Omit --cov-report to generate a report that gitlab can pick up with its log parser COVERAGE: --cov --cov-branch --cov-append @@ -59,20 +59,12 @@ extends: - .depends-full - .unit-tests - variables: - # Hide the RTS parser from py.test as it will break test collection as - # it requires a newer version of Python - PYTEST_IGNORES: --ignore lib/debian/tests/test_repro_deb822.py --ignore lib/debian/_deb822_repro/ unit-tests-oldoldstable: image: debian:oldoldstable extends: - .depends-full - .unit-tests - variables: - # Hide the RTS parser from py.test as it will break test collection as - # it requires a newer version of Python - PYTEST_IGNORES: --ignore lib/debian/tests/test_repro_deb822.py --ignore lib/debian/_deb822_repro/ style: extends: .depends-full diff -Nru python-debian-0.1.44/lib/conftest.py python-debian-0.1.46/lib/conftest.py --- python-debian-0.1.44/lib/conftest.py 1970-01-01 00:00:00.000000000 +0000 +++ python-debian-0.1.46/lib/conftest.py 2022-07-08 16:45:29.000000000 +0000 @@ -0,0 +1,18 @@ +try: + from typing import Dict, Any +except ImportError: + pass + +import pytest + +from debian.tests.stubbed_arch_table import StubbedDpkgArchTable + + +@pytest.fixture(autouse=True) +def doctest_add_load_arch_table(doctest_namespace): + # type: (Dict[str, Any]) -> None + # Provide a custom namespace for doctests such that we can have them use + # a custom environment. Use sparingly. + # - For this to work, the doctests MUST NOT import the names listed here + # (as the import would overwrite the stub) + doctest_namespace['DpkgArchTable'] = StubbedDpkgArchTable diff -Nru python-debian-0.1.44/lib/debian/_arch_table.py python-debian-0.1.46/lib/debian/_arch_table.py --- python-debian-0.1.44/lib/debian/_arch_table.py 1970-01-01 00:00:00.000000000 +0000 +++ python-debian-0.1.46/lib/debian/_arch_table.py 2022-07-08 16:45:29.000000000 +0000 @@ -0,0 +1,405 @@ +"""architecture matching + +This leverages code from dpkg's Dpkg::Arch as well as python rewrites from +other people. Copyright years imported from the sources. + +@copyright: 2006-2015 Guillem Jover +@copyright: 2014, Ansgar Burchardt +@copyright: 2014-2017, Johannes Schauer Marin Rodrigues +@copyright: 2022, Niels Thykier +@license: GPL-2+ +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import os + +try: + from typing import Iterator, Iterable, Optional, IO, List, Dict, Union + from os import PathLike +except ImportError: + pass + +import collections.abc + + +def _parse_table_file(fd): + # type: (IO[str]) -> Iterable[List[str]] + for line in fd: + line = line.rstrip() + if not line or line.startswith("#"): + continue + yield line.split() + + +_QuadTuple = collections.namedtuple("_QuadTuple", ['api_name', 'libc_name', 'os_name', 'cpu_name']) + + +class QuadTupleDpkgArchitecture(_QuadTuple): + """Implementation detail of ArchTable""" + + def __contains__(self, item): + # type: (object) -> bool + if isinstance(item, QuadTupleDpkgArchitecture): + # This covers both equal and wildcard matches and semantically matches how dpkg does it + return self.api_name in ('any', item.api_name) \ + and self.libc_name in ('any', item.libc_name) \ + and self.os_name in ('any', item.os_name) \ + and self.cpu_name in ('any', item.cpu_name) + return super().__contains__(item) + + @property + def is_wildcard(self): + # type: () -> bool + return any(x == 'any' for x in self) + + +class DpkgArchTable: + + def __init__(self, arch2tuple): + # type: (Dict[str, QuadTupleDpkgArchitecture]) -> None + self._arch2table = arch2tuple + self._wildcard_cache = { + 'any': QuadTupleDpkgArchitecture('any', 'any', 'any', 'any') + } # type: Dict[str, QuadTupleDpkgArchitecture] + + @classmethod + def load_arch_table(cls, path='/usr/share/dpkg'): + # type: (Union[str, PathLike[str]]) -> DpkgArchTable + # NOTE! This method is stubbed in including doctests to support non-Debian systems + # See conftest.py for the concrete implementation and the limited data set available. + """Load the Dpkg Architecture Table + + This class method loads the architecture table from dpkg, so it can be used. + + >>> arch_table = DpkgArchTable.load_arch_table() + >>> arch_table.matches_architecture("amd64", "any") + True + + The method assumes the dpkg "tuple arch" format version 1.0 or the older triplet format. + + :param path: Choose a different directory for loading the architecture data. The provided + directory must contain the architecture data files from dpkg (such as "tupletable" and + "cputable") + """ + tupletable_path = os.path.join(path, 'tupletable') + cputable_path = os.path.join(path, 'cputable') + triplet_compat = False + if not os.path.exists(tupletable_path): + triplettable_path = os.path.join(path, 'triplettable') + if os.path.join(triplettable_path): + triplet_compat = True + tupletable_path = triplettable_path + + with open(tupletable_path, encoding='utf-8') as tuple_fd,\ + open(cputable_path, encoding='utf-8') as cpu_fd: + return cls._from_file(tuple_fd, cpu_fd, triplet_compat=triplet_compat) + + @classmethod + def _from_file(cls, tuple_table_fd, cpu_table_fd, triplet_compat=False): + # type: (IO[str], IO[str], bool) -> DpkgArchTable + arch2tuple = {} # type: Dict[str, QuadTupleDpkgArchitecture] + cpu_list = [x[0] for x in _parse_table_file(cpu_table_fd)] + for row in _parse_table_file(tuple_table_fd): + # Manual unpack (so we support new columns) + dpkg_tuple = row[0] + dpkg_arch = row[1] + + if triplet_compat: + dpkg_tuple = "base-" + dpkg_tuple + + if '' in dpkg_tuple: + for cpu_name in cpu_list: + debtuple_cpu = dpkg_tuple.replace('', cpu_name) + dpkg_arch_cpu = dpkg_arch.replace('', cpu_name) + arch2tuple[dpkg_arch_cpu] = QuadTupleDpkgArchitecture( + *debtuple_cpu.split('-', 3) + ) + else: + arch2tuple[dpkg_arch] = QuadTupleDpkgArchitecture(*dpkg_tuple.split('-', 3)) + return DpkgArchTable(arch2tuple) + + def _dpkg_wildcard_to_tuple(self, arch): + # type: (str) -> QuadTupleDpkgArchitecture + try: + return self._wildcard_cache[arch] + except KeyError: + pass + + arch_tuple = arch.split('-', 3) + if 'any' in arch_tuple: + # This loop was written with the wildcard 'any' is always pre-cached. + # (it might still work) + while len(arch_tuple) < 4: + arch_tuple.insert(0, 'any') + result = QuadTupleDpkgArchitecture(*arch_tuple) + else: + result = self._dpkg_arch_to_tuple(arch) + self._wildcard_cache[arch] = result + return result + + def _dpkg_arch_to_tuple(self, dpkg_arch): + # type: (str) -> QuadTupleDpkgArchitecture + if dpkg_arch.startswith("linux-"): + dpkg_arch = dpkg_arch[6:] + + return self._arch2table[dpkg_arch] + + def matches_architecture(self, architecture, alias): + # type: (str, str) -> bool + """Determine if a dpkg architecture matches another architecture or a wildcard [debarch_is] + + This method is the closest match to dpkg's Dpkg::Arch::debarch_is function. + + >>> arch_table = DpkgArchTable.load_arch_table() + >>> arch_table.matches_architecture("amd64", "linux-any") + True + >>> arch_table.matches_architecture("i386", "linux-any") + True + >>> arch_table.matches_architecture("amd64", "amd64") + True + >>> arch_table.matches_architecture("i386", "amd64") + False + >>> arch_table.matches_architecture("all", "amd64") + False + >>> arch_table.matches_architecture("all", "all") + True + >>> # i386 is the short form of linux-i386. Therefore, it does not match kfreebsd-i386 + >>> arch_table.matches_architecture("i386", "kfreebsd-i386") + False + >>> # Note that "armel" and "armhf" are "arm" CPUs, so it is matched by "any-arm" + >>> # (similar holds for some other architecture <-> CPU name combinations) + >>> all(arch_table.matches_architecture(n, 'any-arm') for n in ['armel', 'armhf']) + True + >>> # Since "armel" is not a valid CPU name, this returns False (the correct would be + >>> # any-arm as noted above) + >>> arch_table.matches_architecture("armel", "any-armel") + False + >>> # Wildcards used as architecture always fail (except for special cases noted in the + >>> # compatibility notes below) + >>> arch_table.matches_architecture("any-i386", "i386") + False + >>> # any-i386 is not a subset of linux-any (they only have i386/linux-i386 as overlap) + >>> arch_table.matches_architecture("any-i386", "linux-any") + False + >>> # Compatibility with dpkg - if alias is `any` then it always returns True + >>> # even if the input otherwise would not make sense. + >>> arch_table.matches_architecture("any-unknown", "any") + True + >>> # Another side effect of the dpkg compatibility + >>> arch_table.matches_architecture("all", "any") + True + + Compatibility note: The method emulates Dpkg::Arch::debarch_is function and therefore + returns True if both parameters are the same even though they are wildcards or not known + to be architectures. Additionally, if `alias` is `any`, then this method always returns + True as `any` is the "match-everything-wildcard". + + :param architecture: A string representing a dpkg architecture. + :param alias: A string representing a dpkg architecture or wildcard + to match with. + :returns: True if the `architecture` parameter is (logically) the same as the `alias` + parameter OR, if `alias` is a wildcard, the `architecture` parameter is a + subset of the wildcard. + The method returns False if `architecture` is not a known dpkg architecture, + or it is a wildcard. + """ + if alias in ('any', architecture): + # Dpkg::Arch has this shortcut too, which does not check whether they are valid + # architectures. + return True + try: + dpkg_arch = self._dpkg_arch_to_tuple(architecture) + dpkg_wildcard = self._dpkg_wildcard_to_tuple(alias) + except KeyError: + return False + return dpkg_arch in dpkg_wildcard + + def architecture_equals(self, arch1, arch2): + # type: (str, str) -> bool + """Determine whether two dpkg architecture are exactly the same [debarch_eq] + + Unlike Python's `==` operator, this method also accounts for things like `linux-amd64` is + a valid spelling of the dpkg architecture `amd64` (i.e., + `architecture_equals("linux-amd64", "amd64")` is True). + + This method is the closest match to dpkg's Dpkg::Arch::debarch_eq function. + + >>> arch_table = DpkgArchTable.load_arch_table() + >>> arch_table.architecture_equals("linux-amd64", "amd64") + True + >>> arch_table.architecture_equals("amd64", "linux-i386") + False + >>> arch_table.architecture_equals("i386", "linux-amd64") + False + >>> arch_table.architecture_equals("amd64", "amd64") + True + >>> arch_table.architecture_equals("i386", "amd64") + False + >>> # Compatibility with dpkg: if the parameters are equal, then it always return True + >>> arch_table.architecture_equals("unknown", "unknown") + True + + Compatibility note: The method emulates Dpkg::Arch::debarch_eq function and therefore + returns True if both parameters are the same even though they are wildcards or not known + to be architectures. + + :param arch1: A string representing a dpkg architecture. + :param arch2: A string representing a dpkg architecture. + :returns: True if the dpkg architecture parameters are (logically) the exact same. + """ + if arch1 == arch2: + # Dpkg::Arch has this shortcut too, which does not check whether they are valid + # architectures. + return True + try: + dpkg_arch1 = self._dpkg_arch_to_tuple(arch1) + dpkg_arch2 = self._dpkg_arch_to_tuple(arch2) + except KeyError: + return False + return dpkg_arch1 == dpkg_arch2 + + def architecture_is_concerned(self, architecture, architecture_restrictions, + allow_mixing_positive_and_negative=False, + ): + # type: (str, Iterable[str], bool) -> bool + """Determine if a dpkg architecture is part of a list of restrictions [debarch_is_concerned] + + This method is the closest match to dpkg's Dpkg::Arch::debarch_is_concerned function. + + Compatibility notes: + * The Dpkg::Arch::debarch_is_concerned function allow matching of negative and positive + restrictions by default. Often, this behaviour is not allowed nor recommended and the + Debian Policy does not allow this practice in e.g., Build-Depends. Therefore, this + implementation defaults to raising ValueError when this occurs. If the original + behaviour is needed, set `allow_mixing_positive_and_negative` to True. + * The Dpkg::Arch::debarch_is_concerned function is lazy and exits as soon as it finds a + match. This means that if negative and positive restrictions are mixed, then order of + the matches are important. This adaption matches that behaviour (provided that + `allow_mixing_positive_and_negative` is set to True) + + >>> arch_table = DpkgArchTable.load_arch_table() + >>> arch_table.architecture_is_concerned("linux-amd64", ["amd64", "i386"]) + True + >>> arch_table.architecture_is_concerned("amd64", ["!amd64", "!i386"]) + False + >>> # This is False because the "!amd64" is matched first. + >>> arch_table.architecture_is_concerned("linux-amd64", ["!linux-amd64", "linux-any"], + ... allow_mixing_positive_and_negative=True) + False + >>> # This is True because the "linux-any" is matched first. + >>> arch_table.architecture_is_concerned("linux-amd64", ["linux-any", "!linux-amd64"], + ... allow_mixing_positive_and_negative=True) + True + + :param architecture: A string representing a dpkg architecture/wildcard. + :param architecture_restrictions: A list of positive (amd64) or negative (!amd64) dpkg + architectures or/and wildcards. + :param allow_mixing_positive_and_negative: If True, the `architecture_restrictions` list + can mix positive and negative (e.g., ["!any-amd64", "any"]) + restrictions. If False, mixing will trigger a ValueError. + :returns: True if `architecture` is accepted by the `architecture_restrictions`. + """ + + # Our implementation diverges a bit from the Dpkg::Arch one because we want to enforce + # allow_mixing_positive_and_negative=False even if the input matches before the + # inconsistency is detected. + + verdict = None # type: Optional[bool] + positive_match_seen = False + negative_match_seen = False + arch_restriction_iter = iter(architecture_restrictions) + + try: + dpkg_arch = self._dpkg_arch_to_tuple(architecture) + except KeyError: + return False + + for arch_restriction in arch_restriction_iter: + # Should not happen in practice, but remove the special-case to avoid IndexError + # (dpkg assumes invalid/unknown input does not match, so we can use a "continue" here) + if arch_restriction == '': + continue + + if arch_restriction[0] == '!': + negative_match_seen = True + else: + positive_match_seen = True + + if verdict is not None: + # We already know what the answer is. However, we are running through the remaining + # input to ensure there is mixing of positive and negative restrictions. + continue + + # Blindly matching Dpkg::Arch here, which also forgives uppercase letters. + arch_restriction = arch_restriction.lower() + verdict_if_matched = True + arch_restriction_positive = arch_restriction + + if arch_restriction[0] == '!': + verdict_if_matched = False + arch_restriction_positive = arch_restriction[1:] + + dpkg_wildcard = self._dpkg_wildcard_to_tuple(arch_restriction_positive) + + # Inlined version of self.matches_architecture to reduce the number of lookups + if dpkg_arch in dpkg_wildcard: + verdict = verdict_if_matched + if allow_mixing_positive_and_negative: + # If we do not care about the mixing, then we can closer emulate the dpkg + # implementation by existing early now + return verdict + + if not allow_mixing_positive_and_negative and positive_match_seen and negative_match_seen: + raise ValueError("architecture_restrictions contained mixed positive and negative" + "restrictions (and allow_mixing_positive_and_negative was not True)") + + # If none of the restrictions directly matched the architecture, then this is now + # a question of whether there was a negative match. If there was a negative match, + # then it would have included the input as it is basically "any except " + if verdict is None: + verdict = negative_match_seen + return verdict + + def is_wildcard(self, wildcard): + # type: (str) -> bool + """Determine if a given string is a dpkg wildcard [debarch_is_wildcard] + + This method is the closest match to dpkg's Dpkg::Arch::debarch_is_wildcard function. + + >>> arch_table = DpkgArchTable.load_arch_table() + >>> arch_table.is_wildcard("linux-any") + True + >>> arch_table.is_wildcard("amd64") + False + >>> arch_table.is_wildcard("unknown") + False + >>> # Compatibility with the dpkg version of the function. + >>> arch_table.is_wildcard("unknown-any") + True + + Compatibility note: The original dpkg function does not ensure that the wildcard matches + any supported architecture and this re-implementation matches that behaviour. Therefore, + this method can return True for a wildcard that can never match anything in practice. + + :param wildcard: A string that might represent a dpkg architecture or wildcard. + :returns: True the parameter is a known dpkg wildcard. + """ + try: + dpkg_arch = self._dpkg_wildcard_to_tuple(wildcard) + except KeyError: + return False + else: + # _dpkg_wildcard_to_tuple falls back to concrete architectures so this can be False + return dpkg_arch.is_wildcard diff -Nru python-debian-0.1.44/lib/debian/changelog.py python-debian-0.1.46/lib/debian/changelog.py --- python-debian-0.1.44/lib/debian/changelog.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/changelog.py 2022-07-08 16:45:29.000000000 +0000 @@ -40,7 +40,6 @@ * Some change -- John Doe Wed, 31 Mar 2021 20:31:55 -0000 - If you have the full contents of a changelog, but are only interested in the @@ -900,7 +899,9 @@ block = ChangeBlock(package, version, distributions, urgency, urgency_comment, changes, author, date, other_pairs, encoding) - block.add_trailing_line('') + if self._blocks: + # #998715 - only add a trailing line if there are other blocks. + block.add_trailing_line('') self._blocks.insert(0, block) def write_to_open_file(self, filehandle): diff -Nru python-debian-0.1.44/lib/debian/copyright.py python-debian-0.1.46/lib/debian/copyright.py --- python-debian-0.1.44/lib/debian/copyright.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/copyright.py 2022-07-08 16:45:29.000000000 +0000 @@ -36,6 +36,7 @@ from typing import ( Any, Callable, + FrozenSet, IO, Iterable, Iterator, @@ -45,8 +46,8 @@ Text, Tuple, Union, - TYPE_CHECKING, - ) + TYPE_CHECKING, cast, +) ParagraphTypes = Union["FilesParagraph", "LicenseParagraph"] AllParagraphTypes = Union["Header", "FilesParagraph", "LicenseParagraph"] @@ -54,8 +55,18 @@ # Lack of typing is not important at runtime TYPE_CHECKING = False -from debian import deb822 - +from debian._deb822_repro.parsing import ( + parse_deb822_file, + Deb822ParagraphElement, + Deb822FileElement, Deb822NoDuplicateFieldsParagraphElement, +) +from debian.deb822 import RestrictedField, RestrictedFieldError + +try: + # Typing only + from debian.deb822 import Deb822ValueType +except ImportError: + pass _CURRENT_FORMAT = ( 'https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/') @@ -147,17 +158,15 @@ """ super(Copyright, self).__init__() - self.__paragraphs = [] # type: List[ParagraphTypes] + self.__paragraphs = [] # type: List[AllParagraphTypes] if sequence is not None: - paragraphs = list(deb822.Deb822.iter_paragraphs( - sequence=sequence, encoding=encoding)) - if not paragraphs: - raise NotMachineReadableError('no paragraphs in input') - self.__header = Header(paragraphs[0]) - for i in range(1, len(paragraphs)): - p = paragraphs[i] - if 'Files' in p: + header = None + self.__file = parse_deb822_file(sequence=sequence, encoding=encoding) + for p in self.__file: + if header is None: + header = Header(p) + elif 'Files' in p: pf = FilesParagraph(p, strict) self.__paragraphs.append(pf) elif 'License' in p: @@ -166,9 +175,15 @@ else: _complain('Non-header paragraph has neither "Files" nor ' '"License" fields', strict) + if not header: + raise NotMachineReadableError('no paragraphs in input') + self.__header = header else: + self.__file = Deb822FileElement.new_empty_file() self.__header = Header() + self.__file.append(self.__header._underlying_paragraph) + self.__paragraphs.append(self.__header) @property def header(self): @@ -236,6 +251,7 @@ if isinstance(p, FilesParagraph): last_i = i self.__paragraphs.insert(last_i + 1, paragraph) + self.__file.insert(last_i + 2, paragraph._underlying_paragraph) def all_license_paragraphs(self): # type: () -> Iterator[LicenseParagraph] @@ -251,6 +267,7 @@ if not isinstance(paragraph, LicenseParagraph): raise TypeError('paragraph must be a LicenseParagraph instance') self.__paragraphs.append(paragraph) + self.__file.append(paragraph._underlying_paragraph) def dump(self, f=None): # type: (Optional[IO[Text]]) -> Optional[str] @@ -261,17 +278,13 @@ (i.e. that accepts unicode objects directly). It is thus up to the caller to arrange for the file to do any appropriate encoding. """ - return_string = False - if f is None: - return_string = True - f = io.StringIO() - self.header.dump(f, text_mode=True) - for p in self.__paragraphs: - f.write('\n') - p.dump(f, text_mode=True) - if return_string: - return f.getvalue() # type: ignore - return None + # TODO(jelmer): Write bytes + s = self.__file.dump() + if f is not None: + f.write(s) + return None + return s + def _single_line(s): # type: (str) -> str @@ -359,7 +372,7 @@ def format_multiline(s): # type: (Optional[str]) -> Optional[str] - """Formats multiline text for insertion in a Deb822 field. + """Formats multiline text for insertion in a Deb822ParagraphElement field. Each line except for the first one is prefixed with a single space. Lines that are blank or only whitespace are replaced with ' .' @@ -500,7 +513,176 @@ return re.compile(buf.getvalue(), re.MULTILINE | re.DOTALL) -class FilesParagraph(deb822.RestrictedWrapper): +class _ClassInitMeta(type): + """Metaclass for classes that can be initialized at creation time. + + Implement the method:: + + @classmethod + def _class_init(cls, new_attrs): + pass + + on a class, and apply this metaclass to it. The _class_init method will be + called right after the class is created. The 'new_attrs' param is a dict + containing the attributes added in the definition of the class. + """ + + def __init__(cls, # type: Any + name, # type: Any + bases, # type: Any + attrs, # type: Any + ): + # type (...) -> None + super(_ClassInitMeta, cls).__init__(name, bases, attrs) + cls._class_init(attrs) + + +class _RestrictedWrapper(metaclass=_ClassInitMeta): + """Base class to wrap a Deb822 object, restricting write access to some keys. + + The underlying data is hidden internally. Subclasses may keep a reference + to the data before giving it to this class's constructor, if necessary, but + RestrictedField should cover most use-cases. The dump method from + Deb822 is directly proxied. + + Typical usage:: + + class Foo(object): + def __init__(self, ...): + # ... + + @staticmethod + def from_str(self, s): + # Parse s... + return Foo(...) + + def to_str(self): + # Return in string format. + return ... + + class MyClass(deb822._RestrictedWrapper): + def __init__(self): + data = Deb822ParagraphElement.new_empty_paragraph() + data['Bar'] = 'baz' + super(MyClass, self).__init__(data) + + foo = deb822.RestrictedField( + 'Foo', from_str=Foo.from_str, to_str=Foo.to_str) + + bar = deb822.RestrictedField('Bar', allow_none=False) + + d = MyClass() + d['Bar'] # returns 'baz' + d['Bar'] = 'quux' # raises RestrictedFieldError + d.bar = 'quux' + d.bar # returns 'quux' + d['Bar'] # returns 'quux' + + d.foo = Foo(...) + d['Foo'] # returns string representation of foo + """ + + __restricted_fields = frozenset() # type: FrozenSet[str] + + @classmethod + def _class_init(cls, new_attrs): # type: ignore + restricted_fields = [] + for attr_name, val in new_attrs.items(): + if isinstance(val, RestrictedField): + restricted_fields.append(val.name.lower()) + cls.__init_restricted_field(attr_name, val) # type: ignore + cls.__restricted_fields = frozenset(restricted_fields) + + @classmethod + def __init_restricted_field(cls, attr_name, field): # type: ignore + def getter(self): + # type: (_RestrictedWrapper) -> Deb822ValueType + val = self.__data.get(field.name) + if field.from_str is not None: + return field.from_str(val) + return val + + def setter(self, val): + # type: (_RestrictedWrapper, Deb822ValueType) -> None + if val is not None and field.to_str is not None: + val = field.to_str(val) + if val is None: + if field.allow_none: + if field.name in self.__data: + del self.__data[field.name] + else: + raise TypeError('value must not be None') + else: + self.__data[field.name] = val + + setattr(cls, attr_name, property(getter, setter, None, field.name)) + + def __init__(self, data): + # type: (Deb822ParagraphElement) -> None + """Initializes the wrapper over 'data', a Deb822ParagraphElement object.""" + super(_RestrictedWrapper, self).__init__() + if not isinstance(data, Deb822NoDuplicateFieldsParagraphElement): + raise ValueError("Paragraph has duplicated fields: " + str(data.__class__.__qualname__)) + self.__data = data # type: Deb822NoDuplicateFieldsParagraphElement + + @property + def _underlying_paragraph(self): + # type: () -> Deb822ParagraphElement + return self.__data + + def __getitem__(self, key): + # type: (str) -> Deb822ValueType + return self.__data[key] + + def __setitem__(self, key, value): + # type: (str, Deb822ValueType) -> None + if key.lower() in self.__restricted_fields: + raise RestrictedFieldError( + '%s may not be modified directly; use the associated' + ' property' % key) + self.__data[key] = value + + def __delitem__(self, key): + # type: (str) -> None + if key.lower() in self.__restricted_fields: + raise RestrictedFieldError( + '%s may not be modified directly; use the associated' + ' property' % key) + del self.__data[key] + + def __iter__(self): + # type: () -> Iterable[str] + return iter(self.__data) + + def __len__(self): + # type: () -> int + return len(self.__data) + + def dump(self, + fd=None, # type: Optional[Union[IO[str], IO[bytes]]] + encoding=None, # type: Optional[str] + text_mode=False, # type: bool + ): + # type: (...) -> Optional[str] + """Calls dump() on the underlying data object. + + See Deb822.dump for more information. + """ + if fd is not None: + if encoding is None and not text_mode: + self.__data.dump(cast('IO[bytes]', fd)) + return None + # Compat with Deb822's dump + as_str = self.__data.dump() + if encoding is not None: + cast('IO[bytes]', fd).write(as_str.encode(encoding)) + elif text_mode: + cast('IO[str]', fd).write(as_str) + return None + return self.__data.dump() + + +class FilesParagraph(_RestrictedWrapper): """Represents a Files paragraph of a debian/copyright file. This kind of paragraph is used to specify the copyright and license for a @@ -510,7 +692,7 @@ _default_re = re.compile('') def __init__(self, data, _internal_validate=True, strict=True): - # type: (deb822.Deb822, bool, bool) -> None + # type: (Deb822ParagraphElement, bool, bool) -> None super(FilesParagraph, self).__init__(data) if _internal_validate: @@ -540,7 +722,7 @@ :param license: The Licence for the files. """ # pylint: disable=redefined-builtin - p = cls(deb822.Deb822(), _internal_validate=False) + p = cls(Deb822ParagraphElement.new_empty_paragraph(), _internal_validate=False) # mypy doesn't handle the metaprogrammed properties at all p.files = files # type: ignore p.copyright = copyright # type: ignore @@ -568,20 +750,20 @@ return False return pat.match(filename) is not None - files = deb822.RestrictedField( + files = RestrictedField( 'Files', from_str=_SpaceSeparated.from_str, to_str=_SpaceSeparated.to_str, allow_none=False) - copyright = deb822.RestrictedField('Copyright', allow_none=False) + copyright = RestrictedField('Copyright', allow_none=False) - license = deb822.RestrictedField( + license = RestrictedField( 'License', from_str=License.from_str, to_str=License.to_str, allow_none=False) - comment = deb822.RestrictedField('Comment') + comment = RestrictedField('Comment') -class LicenseParagraph(deb822.RestrictedWrapper): +class LicenseParagraph(_RestrictedWrapper): """Represents a standalone license paragraph of a debian/copyright file. Minimally, this kind of paragraph requires a 'License' field and has no @@ -590,7 +772,7 @@ """ def __init__(self, data, _internal_validate=True): - # type: (deb822.Deb822, bool) -> None + # type: (Deb822ParagraphElement, bool) -> None super(LicenseParagraph, self).__init__(data) if _internal_validate: if 'License' not in data: @@ -606,24 +788,24 @@ # pylint: disable=redefined-builtin if not isinstance(license, License): raise TypeError('license must be a License instance') - paragraph = cls(deb822.Deb822(), _internal_validate=False) + paragraph = cls(Deb822ParagraphElement.new_empty_paragraph(), _internal_validate=False) paragraph.license = license # type: ignore ## properties return paragraph # TODO(jsw): Validate that the synopsis of the license is a short name or # short name with exceptions (not an alternatives expression). This # requires help from the License class. - license = deb822.RestrictedField( + license = RestrictedField( 'License', from_str=License.from_str, to_str=License.to_str, allow_none=False) - comment = deb822.RestrictedField('Comment') + comment = RestrictedField('Comment') # Hide 'Files'. - __files = deb822.RestrictedField('Files') # pylint: disable=unused-private-member + __files = RestrictedField('Files') -class Header(deb822.RestrictedWrapper): +class Header(_RestrictedWrapper): """Represents the header paragraph of a debian/copyright file. Property values are all immutable, such that in order to modify them you @@ -631,14 +813,14 @@ """ def __init__(self, data=None): - # type: (Optional[deb822.Deb822]) -> None + # type: (Optional[Deb822ParagraphElement]) -> None """Initializer. - :param data: A deb822.Deb822 object for underlying data. If None, a + :param data: A Deb822ParagraphElement object for underlying data. If None, a new one will be created. """ if data is None: - data = deb822.Deb822() + data = Deb822ParagraphElement.new_empty_paragraph() data['Format'] = _CURRENT_FORMAT if 'Format-Specification' in data: @@ -681,31 +863,31 @@ return self.format == _CURRENT_FORMAT # type: ignore # lots of type ignores due to https://github.com/python/mypy/issues/1279 - format = deb822.RestrictedField( + format = RestrictedField( 'Format', to_str=_single_line, allow_none=False) - upstream_name = deb822.RestrictedField( + upstream_name = RestrictedField( 'Upstream-Name', to_str=_single_line) - upstream_contact = deb822.RestrictedField( + upstream_contact = RestrictedField( 'Upstream-Contact', from_str=_LineBased.from_str, to_str=_LineBased.to_str) - source = deb822.RestrictedField('Source') + source = RestrictedField('Source') - disclaimer = deb822.RestrictedField('Disclaimer') + disclaimer = RestrictedField('Disclaimer') - comment = deb822.RestrictedField('Comment') + comment = RestrictedField('Comment') - license = deb822.RestrictedField( + license = RestrictedField( 'License', from_str=License.from_str, to_str=License.to_str) - copyright = deb822.RestrictedField('Copyright') + copyright = RestrictedField('Copyright') - files_excluded = deb822.RestrictedField( + files_excluded = RestrictedField( 'Files-Excluded', from_str=_LineBased.from_str, to_str=_LineBased.to_str) - files_included = deb822.RestrictedField( + files_included = RestrictedField( 'Files-Included', from_str=_LineBased.from_str, to_str=_LineBased.to_str) diff -Nru python-debian-0.1.44/lib/debian/deb822.py python-debian-0.1.46/lib/debian/deb822.py --- python-debian-0.1.44/lib/debian/deb822.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/deb822.py 2022-07-08 16:45:29.000000000 +0000 @@ -228,10 +228,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -import collections import collections.abc import datetime import email.utils +import functools import logging import io import re @@ -339,6 +339,13 @@ return False +# In Python 3.10, there is a default of 128. Python 3.5 requires an explicit cache size. +@functools.lru_cache(128) +def _cached_strI(v): + # type: (str) -> _strI + return _strI(v) + + GPGV_DEFAULT_KEYRINGS = frozenset(['/usr/share/keyrings/debian-keyring.gpg']) GPGV_EXECUTABLE = '/usr/bin/gpgv' @@ -467,9 +474,9 @@ if _parsed is not None: self.__parsed = _parsed if _fields is None: - self.__keys.extend([_strI(k) for k in self.__parsed]) + self.__keys.extend([_cached_strI(k) for k in self.__parsed]) else: - self.__keys.extend([_strI(f) for f in _fields if f in self.__parsed]) + self.__keys.extend([_cached_strI(f) for f in _fields if f in self.__parsed]) # ### BEGIN collections.abc.MutableMapping methods @@ -484,7 +491,8 @@ def __setitem__(self, key, value): # type: (str, Deb822ValueType) -> None - keyi = _strI(key) + # The `_cached_strI` pays off in the long run (with Packages files or similar sized files) + keyi = _cached_strI(key) self.__keys.add(keyi) self.__dict[keyi] = value @@ -595,7 +603,7 @@ class Deb822(Deb822Dict): """ Generic Deb822 data - :param sequence: a string, or any any object that returns a line of + :param sequence: a string, or any object that returns a line of input each time, normally a file. Alternately, sequence can be a dict that contains the initial key-value pairs. When python-apt is present, sequence can also be a compressed object, @@ -754,40 +762,30 @@ @staticmethod def _skip_useless_lines(sequence): - # type: (IterableInputDataType) -> Union[Iterator[bytes], Iterator[str]] + # type: (IterableInputDataType) -> Union[Iterator[bytes]] """Yields only lines that do not begin with '#'. Also skips any blank lines at the beginning of the input. """ at_beginning = True for line in sequence: - # The bytes/str polymorphism required here to support Python 3 - # is unpleasant, but fortunately limited. We need this because - # at this point we might have been given either bytes or - # Unicode, and we haven't yet got to the point where we can try - # to decode a whole paragraph and detect its encoding. - if isinstance(line, bytes): - if line.startswith(b'#'): - continue - else: - if line.startswith('#'): - continue + # _skip_useless_lines is only called before one place and that prefers + # bytes, so we can just convert the input into bytes and simplify + # our checks. + if isinstance(line, str): + line = line.encode() + if line.startswith(b'#'): + continue if at_beginning: - if isinstance(line, bytes): - if not line.rstrip(b'\r\n'): - continue - else: - if not line.rstrip('\r\n'): - continue + if not line.rstrip(b'\r\n'): + continue at_beginning = False yield line # regexps for parsing the Deb822 data # The key is non-whitespace, non-colon characters before any colon. _key_part = r"^(?P[^: \t\n\r\f\v]+)\s*:\s*" - _single = re.compile(_key_part + r"(?P\S.*?)\s*$") - _multi = re.compile(_key_part + r"$") - _multidata = re.compile(r"^\s(?P.+?)\s*$") + _new_field_re = re.compile(_key_part + r"(?P(?:\S+(\s+\S+)*)?)\s*$") # Explicit source entries in the file can be either: # Source: source_package @@ -818,35 +816,23 @@ self._skip_useless_lines(sequence), strict): line = self.decoder.decode(linebytes) - m = self._single.match(line) + m = self._new_field_re.match(line) if m: if curkey: self[curkey] = content - if not wanted_field(m.group('key')): - curkey = None - continue - curkey = m.group('key') - content = m.group('data') - continue - - m = self._multi.match(line) - if m: - if curkey: - self[curkey] = content - if not wanted_field(m.group('key')): + if not wanted_field(curkey): curkey = None continue - curkey = m.group('key') - content = "" + content = m.group('data') continue - m = self._multidata.match(line) - if m: - content += '\n' + line # XXX not m.group('data')? + # Skip lines that entirely whitespace + if line and line[0].isspace() and not line.isspace(): + content += '\n' + line continue if curkey: @@ -959,7 +945,7 @@ text_mode=False, # type: bool ): # type: (...) -> Optional[str] - """Dump the the contents in the original format + """Dump the contents in the original format :param fd: file-like object to which the data should be written (see notes below) @@ -1097,9 +1083,6 @@ # regexps for finding the gpg header around signed data _gpgre = re.compile(br'^-----(?PBEGIN|END) ' br'PGP (?P[^-]+)-----[\r\t ]*$') - _initial_blank_line = re.compile(br'^\s*$') - _blank_line_whitespace = re.compile(br'^\s*$') - _blank_line_no_whitespace = re.compile(br'^$') @staticmethod def split_gpg_and_payload(sequence, # type: Union[Iterator[bytes], Iterator[str]] @@ -1130,10 +1113,7 @@ # Include whitespace-only lines in blank lines to split paragraphs. # (see #715558) - if strict.get('whitespace-separates-paragraphs', True): - blank_line = Deb822._blank_line_whitespace - else: - blank_line = Deb822._blank_line_no_whitespace + accept_empty_or_whitespace = strict.get('whitespace-separates-paragraphs', True) first_line = True for line_ in sequence: @@ -1150,15 +1130,19 @@ # skip initial blank lines, if any if first_line: - if Deb822._initial_blank_line.match(line): + if not line or line.isspace(): continue first_line = False - m = Deb822._gpgre.match(line) + m = Deb822._gpgre.match(line) if line.startswith(b'-') else None + # We unconditionally compute whether it is a blank line. We need it for all lines + # that are not GPG lines (which is the vast major of lines). For Packages files, + # using this "simple" solution is about 5% faster than regexes. + is_empty_line = not line or line.isspace() if accept_empty_or_whitespace else not line if not m: if state == b'SAFE': - if not blank_line.match(line): + if not is_empty_line: lines.append(line) else: if not gpg_pre_lines: @@ -1166,7 +1150,7 @@ # this blank line break elif state == b'SIGNED MESSAGE': - if blank_line.match(line): + if is_empty_line: state = b'SAFE' else: gpg_pre_lines.append(line) @@ -1178,7 +1162,7 @@ elif m.group('action') == b'END': gpg_post_lines.append(line) break - if not blank_line.match(line): + if not is_empty_line: if not lines: gpg_pre_lines.append(line) else: @@ -1233,10 +1217,15 @@ if value.endswith('\n'): raise ValueError("value must not end in '\\n'") + if '\n' not in value: + return + # Make sure there are no blank lines (actually, the first one is # allowed to be blank, but no others), and each subsequent line starts # with whitespace - for line in value.splitlines()[1:]: + for no, line in enumerate(value.splitlines()): + if no == 0: + continue if not line: raise ValueError("value must not have blank lines") if not line[0].isspace(): @@ -2418,30 +2407,6 @@ return debian.debian_support.Version(version) -class _ClassInitMeta(type): - """Metaclass for classes that can be initialized at creation time. - - Implement the method:: - - @classmethod - def _class_init(cls, new_attrs): - pass - - on a class, and apply this metaclass to it. The _class_init method will be - called right after the class is created. The 'new_attrs' param is a dict - containing the attributes added in the definition of the class. - """ - - def __init__(cls, # type: Any - name, # type: Any - bases, # type: Any - attrs, # type: Any - ): - # type (...) -> None - super(_ClassInitMeta, cls).__init__(name, bases, attrs) - cls._class_init(attrs) - - class RestrictedField(collections.namedtuple( 'RestrictedField', 'name from_str to_str allow_none')): """Placeholder for a property providing access to a restricted field. @@ -2477,134 +2442,6 @@ cls, name, from_str=from_str, to_str=to_str, allow_none=allow_none) - -class RestrictedWrapper(metaclass=_ClassInitMeta): - """Base class to wrap a Deb822 object, restricting write access to some keys. - - The underlying data is hidden internally. Subclasses may keep a reference - to the data before giving it to this class's constructor, if necessary, but - RestrictedField should cover most use-cases. The dump method from - Deb822 is directly proxied. - - Typical usage:: - - class Foo(object): - def __init__(self, ...): - # ... - - @staticmethod - def from_str(self, s): - # Parse s... - return Foo(...) - - def to_str(self): - # Return in string format. - return ... - - class MyClass(deb822.RestrictedWrapper): - def __init__(self): - data = deb822.Deb822() - data['Bar'] = 'baz' - super(MyClass, self).__init__(data) - - foo = deb822.RestrictedField( - 'Foo', from_str=Foo.from_str, to_str=Foo.to_str) - - bar = deb822.RestrictedField('Bar', allow_none=False) - - d = MyClass() - d['Bar'] # returns 'baz' - d['Bar'] = 'quux' # raises RestrictedFieldError - d.bar = 'quux' - d.bar # returns 'quux' - d['Bar'] # returns 'quux' - - d.foo = Foo(...) - d['Foo'] # returns string representation of foo - """ - - __restricted_fields = frozenset() # type: FrozenSet[str] - - @classmethod - def _class_init(cls, new_attrs): # type: ignore - restricted_fields = [] - for attr_name, val in new_attrs.items(): - if isinstance(val, RestrictedField): - restricted_fields.append(val.name.lower()) - cls.__init_restricted_field(attr_name, val) # type: ignore - cls.__restricted_fields = frozenset(restricted_fields) - - @classmethod - def __init_restricted_field(cls, attr_name, field): # type: ignore - def getter(self): - # type: (RestrictedWrapper) -> Deb822ValueType - val = self.__data.get(field.name) - if field.from_str is not None: - return field.from_str(val) - return val - - def setter(self, val): - # type: (RestrictedWrapper, Deb822ValueType) -> None - if val is not None and field.to_str is not None: - val = field.to_str(val) - if val is None: - if field.allow_none: - if field.name in self.__data: - del self.__data[field.name] - else: - raise TypeError('value must not be None') - else: - self.__data[field.name] = val - - setattr(cls, attr_name, property(getter, setter, None, field.name)) - - def __init__(self, data): - # type: (Deb822) -> None - """Initializes the wrapper over 'data', a Deb822 object.""" - super(RestrictedWrapper, self).__init__() - self.__data = data # type: Deb822 - - def __getitem__(self, key): - # type: (str) -> Deb822ValueType - return self.__data[key] - - def __setitem__(self, key, value): - # type: (str, Deb822ValueType) -> None - if key.lower() in self.__restricted_fields: - raise RestrictedFieldError( - '%s may not be modified directly; use the associated' - ' property' % key) - self.__data[key] = value - - def __delitem__(self, key): - # type: (str) -> None - if key.lower() in self.__restricted_fields: - raise RestrictedFieldError( - '%s may not be modified directly; use the associated' - ' property' % key) - del self.__data[key] - - def __iter__(self): - # type: () -> Iterable[str] - return iter(self.__data) - - def __len__(self): - # type: () -> int - return len(self.__data) - - def dump(self, - fd=None, # type: Optional[Union[IO[str], IO[bytes]]] - encoding=None, # type: Optional[str] - text_mode=False, # type: bool - ): - # type: (...) -> Optional[str] - """Calls dump() on the underlying data object. - - See Deb822.dump for more information. - """ - return self.__data.dump(fd, encoding, text_mode) - - class Removals(Deb822): """Represent an ftp-master removals.822 file diff -Nru python-debian-0.1.44/lib/debian/_deb822_repro/__init__.py python-debian-0.1.46/lib/debian/_deb822_repro/__init__.py --- python-debian-0.1.44/lib/debian/_deb822_repro/__init__.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/_deb822_repro/__init__.py 2022-07-08 16:45:29.000000000 +0000 @@ -1,4 +1,4 @@ -# The "from X import Y as Y" looks weird but we are stuck in a fight +# The "from X import Y as Y" looks weird, but we are stuck in a fight # between mypy and pylint in the CI. # # mypy --strict insists on either of following for re-exporting @@ -39,7 +39,7 @@ ... # Inline comment (associated with the next line) ... libbar, ... ''' - >>> deb822_file = parse_deb822_file(example_deb822_paragraph.splitlines(keepends=True)) + >>> deb822_file = parse_deb822_file(example_deb822_paragraph.splitlines()) >>> paragraph = next(iter(deb822_file)) >>> paragraph['Section'] = 'devel' >>> output = deb822_file.dump() @@ -96,6 +96,8 @@ >>> import contextlib >>> @contextlib.contextmanager ... def open_input(): + ... # Works with and without keepends=True. + ... # Keep the ends here to truly emulate an open file. ... yield dctrl_input.splitlines(keepends=True) >>> def open_output(): ... return open('/dev/null', 'wb') diff -Nru python-debian-0.1.44/lib/debian/_deb822_repro/parsing.py python-debian-0.1.46/lib/debian/_deb822_repro/parsing.py --- python-debian-0.1.44/lib/debian/_deb822_repro/parsing.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/_deb822_repro/parsing.py 2022-07-08 16:45:29.000000000 +0000 @@ -2,19 +2,19 @@ import collections.abc import contextlib +import sys import textwrap import weakref from abc import ABC from types import TracebackType from weakref import ReferenceType +from debian._deb822_repro._util import (combine_into_replacement, BufferingIterator, + len_check_iterator, + ) from debian._deb822_repro.formatter import ( FormatterContentToken, one_value_per_line_trailing_separator, format_field, ) -from debian._util import ( - resolve_ref, LinkedList, LinkedListNode, OrderedSet, _strI, default_field_sort_key, -) -from debian._deb822_repro.types import AmbiguousDeb822FieldKeyError from debian._deb822_repro.tokens import ( Deb822Token, Deb822ValueToken, Deb822SemanticallySignificantWhiteSpace, Deb822SpaceSeparatorToken, Deb822CommentToken, Deb822WhitespaceToken, @@ -22,9 +22,10 @@ Deb822FieldNameToken, Deb822FieldSeparatorToken, Deb822ErrorToken, tokenize_deb822_file, comma_split_tokenizer, whitespace_split_tokenizer, ) -from debian._deb822_repro._util import (combine_into_replacement, BufferingIterator, - len_check_iterator, - ) +from debian._deb822_repro.types import AmbiguousDeb822FieldKeyError +from debian._util import ( + resolve_ref, LinkedList, LinkedListNode, OrderedSet, _strI, default_field_sort_key, +) try: from typing import ( @@ -38,9 +39,14 @@ ParagraphKey, TokenOrElement, Commentish, ParagraphKeyBase, FormatterCallback, ) - StreamingValueParser = Callable[[Deb822Token, BufferingIterator[Deb822Token]], VE] - StrToValueParser = Callable[[str], Iterable[Union['Deb822Token', VE]]] - KVPNode = LinkedListNode['Deb822KeyValuePairElement'] + if TYPE_CHECKING: + StreamingValueParser = Callable[[Deb822Token, BufferingIterator[Deb822Token]], VE] + StrToValueParser = Callable[[str], Iterable[Union['Deb822Token', VE]]] + KVPNode = LinkedListNode['Deb822KeyValuePairElement'] + else: + StreamingValueParser = None + StrToValueParser = None + KVPNode = None except ImportError: if not TYPE_CHECKING: cast = lambda t, v: v @@ -127,8 +133,24 @@ self._node = None +if sys.version_info >= (3, 8) or TYPE_CHECKING: + _Deb822ParsedTokenList_ContextManager = contextlib.AbstractContextManager[T] +else: + # Python 3.5 - 3.7 compat - we are not allowed to subscript the abc.Iterator + # - use this little hack to work around it + # Note that Python 3.5 is so old that it does not have AbstractContextManager, + # so we re-implement it here. + class _Deb822ParsedTokenList_ContextManager(Generic[T]): + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return None + + class Deb822ParsedTokenList(Generic[VE, ST], - contextlib.AbstractContextManager['Deb822ParsedTokenList[VE, ST]'] + _Deb822ParsedTokenList_ContextManager['Deb822ParsedTokenList[VE, ST]'] ): def __init__(self, @@ -471,6 +493,13 @@ if force_reformat: self._changed = True + def clear(self): + # type: () -> None + """Like list.clear() - removes all content (including comments and spaces)""" + if self._token_list: + self._changed = True + self._token_list.clear() + def _iter_content_as_tokens(self): # type: () -> Iterable[Deb822Token] for te in self._token_list: @@ -517,27 +546,35 @@ field_name = kvpair_element.field_name token_list = self._token_list tail = token_list.tail + had_tokens = False for t in self._iter_content_as_tokens(): + had_tokens = True if not t.is_comment and not t.is_whitespace: break else: - raise ValueError("Field must have content (i.e. non-whitespace and non-comments)") - - assert tail is not None - if isinstance(tail, Deb822Token) and tail.is_comment: - raise ValueError("Fields must not end on a comment") - if not tail.convert_to_text().endswith("\n"): - # Always end on a newline - self.append_newline() + if had_tokens: + raise ValueError("Field must be completely empty or have content " + "(i.e. non-whitespace and non-comments)") + if tail is not None: + if isinstance(tail, Deb822Token) and tail.is_comment: + raise ValueError("Fields must not end on a comment") + if not tail.convert_to_text().endswith("\n"): + # Always end on a newline + self.append_newline() + + if self._format_preserve_original_formatting: + value_text = self._generate_field_content() + text = ':'.join((field_name, value_text)) + else: + text = self._generate_reformatted_field_content() - if self._format_preserve_original_formatting: - value_text = self._generate_field_content() - text = ':'.join((field_name, value_text)) + new_content = text.splitlines(keepends=True) else: - text = self._generate_reformatted_field_content() - - new_content = text.splitlines(keepends=True) + # Special-case for the empty list which will be mapped to + # an empty field. Always end on a newline (avoids errors + # if there is a field after this) + new_content = [field_name + ":\n"] # As absurd as it might seem, it is easier to just use the parser to # construct the AST correctly @@ -555,7 +592,7 @@ def sort_elements(self, *, key=None, # type: Optional[Callable[[VE], Any]] - reverse=False, # type: bool + reverse=False # type: bool ): # type: (...) -> None """Sort the elements (abstract values) in this list. @@ -636,8 +673,8 @@ def sort(self, *, key=None, # type: Optional[Callable[[str], Any]] - **kwargs, # type: Any - ): + **kwargs # type: Any + ): # type: (...) -> None """Sort the values (rendered as str) in this list. @@ -662,7 +699,7 @@ def interpret(self, kvpair_element, # type: Deb822KeyValuePairElement - discard_comments_on_read=True, # type: bool + discard_comments_on_read=True # type: bool ): # type: (...) -> T raise NotImplementedError # pragma: no cover @@ -672,7 +709,7 @@ def __init__(self, tokenizer, # type: Callable[[str], Iterable['Deb822Token']] - value_parser, # type: StreamingValueParser[VE] + value_parser # type: StreamingValueParser[VE] ): # type: (...) -> None super().__init__() @@ -682,13 +719,13 @@ def _high_level_interpretation(self, kvpair_element, # type: Deb822KeyValuePairElement token_list, # type: List['TokenOrElement'] - discard_comments_on_read=True, # type: bool + discard_comments_on_read=True # type: bool ): # type: (...) -> T raise NotImplementedError # pragma: no cover def _parse_stream(self, - buffered_iterator, # type: BufferingIterator[Deb822Token] + buffered_iterator # type: BufferingIterator[Deb822Token] ): # type: (...) -> Iterable[Union[Deb822Token, VE]] @@ -701,7 +738,7 @@ def _parse_kvpair( self, - kvpair, # type: Deb822KeyValuePairElement + kvpair # type: Deb822KeyValuePairElement ): # type: (...) -> Iterable[Union[Deb822Token, VE]] content = kvpair.value_element.convert_to_text() @@ -720,7 +757,7 @@ def interpret(self, kvpair_element, # type: Deb822KeyValuePairElement - discard_comments_on_read=True, # type: bool + discard_comments_on_read=True # type: bool ): # type: (...) -> T token_list = [] # type: List['TokenOrElement'] @@ -775,7 +812,7 @@ vtype, # type: Type[VE] stype, # type: Type[ST] default_separator_factory, # type: Callable[[], ST] - render_factory, # type: Callable[[bool], Callable[[VE], str]] + render_factory # type: Callable[[bool], Callable[[VE], str]] ): # type: (...) -> None super().__init__(tokenizer, value_parser) @@ -787,7 +824,7 @@ def _high_level_interpretation(self, kvpair_element, # type: Deb822KeyValuePairElement token_list, # type: List['TokenOrElement'] - discard_comments_on_read=True, # type: bool + discard_comments_on_read=True # type: bool ): # type: (...) -> Deb822ParsedTokenList[VE, ST] return Deb822ParsedTokenList( @@ -908,7 +945,7 @@ yield part def iter_recurse(self, *, - only_element_or_token_type=None, # type: Optional[Type[TE]] + only_element_or_token_type=None # type: Optional[Type[TE]] ): # type: (...) -> Iterable[TE] for part in self.iter_parts(): @@ -979,7 +1016,7 @@ trailing_whitespace_token, # type: Optional[Deb822WhitespaceToken] # only optional if it is the last line of the file and the file does not # end with a newline. - newline_token, # type: Optional[Deb822WhitespaceToken] + newline_token # type: Optional[Deb822WhitespaceToken] ): # type: (...) -> None super().__init__() @@ -1150,7 +1187,7 @@ comment_element, # type: Optional[Deb822CommentElement] field_token, # type: Deb822FieldNameToken separator_token, # type: Deb822FieldSeparatorToken - value_element, # type: Deb822ValueElement + value_element # type: Deb822ValueElement ): # type: (...) -> None super().__init__() @@ -1184,7 +1221,7 @@ def interpret_as(self, interpreter, # type: Interpretation[T] - discard_comments_on_read=True, # type: bool + discard_comments_on_read=True # type: bool ): # type: (...) -> T return interpreter.interpret(self, discard_comments_on_read=discard_comments_on_read) @@ -1230,7 +1267,7 @@ def _unpack_key(item, # type: ParagraphKey - raise_if_indexed=False, # type: bool + raise_if_indexed=False # type: bool ): # type: (...) -> Tuple[_strI, Optional[int], Optional[Deb822FieldNameToken]] index = None # type: Optional[int] @@ -1257,7 +1294,7 @@ def _convert_value_lines_to_lines(value_lines, # type: Iterable[Deb822ValueLineElement] - strip_comments, # type: bool + strip_comments # type: bool ): # type: (...) -> Iterable[str] if not strip_comments: @@ -1268,10 +1305,19 @@ if not x.is_comment) +if sys.version_info >= (3, 8) or TYPE_CHECKING: + _ParagraphMapping_Base = collections.abc.Mapping[ParagraphKey, T] +else: + # Python 3.5 - 3.7 compat - we are not allowed to subscript the abc.Iterator + # - use this little hack to work around it + class _ParagraphMapping_Base(collections.abc.Mapping, Generic[T], ABC): + pass + + # Deb822ParagraphElement uses this Mixin (by having `_paragraph` return self). # Therefore the Mixin needs to call the "proper" methods on the paragraph to # avoid doing infinite recursion. -class AutoResolvingMixin(Generic[T], collections.abc.Mapping[ParagraphKey, T]): +class AutoResolvingMixin(Generic[T], _ParagraphMapping_Base[T]): @property def _auto_resolve_ambiguous_fields(self): @@ -1317,7 +1363,6 @@ # Therefore the Mixin needs to call the "proper" methods on the paragraph to # avoid doing infinite recursion. class Deb822ParagraphToStrWrapperMixin(AutoResolvingMixin[str], - collections.abc.MutableMapping[ParagraphKey, str], ABC): @property @@ -1403,9 +1448,12 @@ field_comment=comment, ) return - # Regenerate the first line with normalized whitespace + # Regenerate the first line with normalized whitespace if necessary first_line, rest = value.split("\n", 1) - value = "".join((" ", first_line.strip(), "\n", rest)) + if first_line and first_line[:1] not in ('\t', ' '): + value = "".join((" ", first_line.strip(), "\n", rest)) + else: + value = "".join((first_line, "\n", rest)) if not value.endswith("\n"): if not self._auto_map_final_newline_in_multiline_values: raise ValueError("Values must end with a newline (or be single line" @@ -1430,7 +1478,7 @@ paragraph, # type: Deb822ParagraphElement *, auto_resolve_ambiguous_fields=False, # type: bool - discard_comments_on_read=True, # type: bool + discard_comments_on_read=True # type: bool ): # type: (...) -> None self.__paragraph = paragraph @@ -1460,8 +1508,9 @@ interpretation, # type: Interpretation[T] *, auto_resolve_ambiguous_fields=False, # type: bool - discard_comments_on_read=True, # type: bool - ) -> None: + discard_comments_on_read=True # type: bool + ): + # type: (...) -> None super().__init__(paragraph, auto_resolve_ambiguous_fields=auto_resolve_ambiguous_fields, discard_comments_on_read=discard_comments_on_read, @@ -1483,7 +1532,7 @@ auto_map_initial_line_whitespace=True, # type: bool auto_resolve_ambiguous_fields=False, # type: bool preserve_field_comments_on_field_updates=True, # type: bool - auto_map_final_newline_in_multiline_values=True, # type: bool + auto_map_final_newline_in_multiline_values=True # type: bool ): # type: (...) -> None super().__init__(paragraph, @@ -1550,7 +1599,7 @@ def as_interpreted_dict_view(self, interpretation, # type: Interpretation[T] *, - auto_resolve_ambiguous_fields=True, # type: bool + auto_resolve_ambiguous_fields=True # type: bool ): # type: (...) -> Deb822InterpretingParagraphWrapper[T] r"""Provide a Dict-like view of the paragraph @@ -1570,7 +1619,7 @@ ... arm64 ... armel ... ''' - >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines(keepends=True)) + >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines()) >>> paragraph = next(iter(dfile)) >>> list_view = paragraph.as_interpreted_dict_view(LIST_SPACE_SEPARATED_INTERPRETATION) >>> # With the defaults, you only deal with the semantic values @@ -1633,7 +1682,7 @@ auto_map_initial_line_whitespace=True, # type: bool auto_resolve_ambiguous_fields=True, # type: bool preserve_field_comments_on_field_updates=True, # type: bool - auto_map_final_newline_in_multiline_values=True, # type: bool + auto_map_final_newline_in_multiline_values=True # type: bool ): # type: (...) -> Deb822DictishParagraphWrapper r"""Provide a Dict[str, str]-like view of this paragraph with non-standard parameters @@ -1648,7 +1697,7 @@ ... # Inline comment (associated with the next line) ... libbar, ... ''' - >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines(keepends=True)) + >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines()) >>> paragraph = next(iter(dfile)) >>> # With the defaults, you only deal with the semantic values >>> # - no leading or trailing whitespace on the first part of the value @@ -1675,12 +1724,12 @@ another value >>> # The comment is present (in case you where wondering) >>> print(paragraph.get_kvpair_element('Bar').convert_to_text(), end='') - Bar: bar + Bar: bar #Comment another value >>> # On the other hand, you can choose to see the values as they are >>> # - We will just reset the paragraph as a "nothing up my sleeve" - >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines(keepends=True)) + >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines()) >>> paragraph = next(iter(dfile)) >>> nonstd_dictview = paragraph.configured_view( ... discard_comments_on_read=False, @@ -1820,7 +1869,7 @@ def get_kvpair_element(self, item, # type: ParagraphKey - use_get=False, # type: bool + use_get=False # type: bool ): # type: (...) -> Optional[Deb822KeyValuePairElement] raise NotImplementedError # pragma: no cover @@ -1834,7 +1883,7 @@ raise NotImplementedError # pragma: no cover def sort_fields(self, - key=None, # type: Optional[Callable[[str], Any]] + key=None # type: Optional[Callable[[str], Any]] ): # type: (...) -> None """Re-order all fields @@ -1850,7 +1899,7 @@ simple_value, # type: str *, preserve_original_field_comment=None, # type: Optional[bool] - field_comment=None, # type: Optional[Commentish] + field_comment=None # type: Optional[Commentish] ): # type: (...) -> None r"""Sets a field in this paragraph to a simple "word" or "phrase" @@ -1864,7 +1913,7 @@ >>> example_deb822_paragraph = ''' ... Package: foo ... ''' - >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines(keepends=True)) + >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines()) >>> p = next(iter(dfile)) >>> p.set_field_to_simple_value("Package", "mscgen") >>> p.set_field_to_simple_value("Architecture", "linux-any kfreebsd-any", @@ -1903,9 +1952,14 @@ raise ValueError("Cannot use set_field_to_simple_value for values with newlines") # Reformat it with a leading space and trailing newline. The latter because it is - # necessary if there any fields after it and the former because it looks nicer so + # necessary if there are any fields after it and the former because it looks nicer so # have single space after the field separator - raw_value = ' ' + simple_value.strip() + "\n" + stripped = simple_value.strip() + if stripped: + raw_value = ' ' + stripped + "\n" + else: + # Special-case for empty values + raw_value = "\n" self.set_field_from_raw_string( item, raw_value, @@ -1918,7 +1972,7 @@ raw_string_value, # type: str *, preserve_original_field_comment=None, # type: Optional[bool] - field_comment=None, # type: Optional[Commentish] + field_comment=None # type: Optional[Commentish] ): # type: (...) -> None """Sets a field in this paragraph to a given text value @@ -1933,7 +1987,7 @@ >>> example_deb822_paragraph = ''' ... Package: foo ... ''' - >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines(keepends=True)) + >>> dfile = parse_deb822_file(example_deb822_paragraph.splitlines()) >>> p = next(iter(dfile)) >>> raw_value = ''' ... Build-Depends: debhelper-compat (= 12), @@ -1977,7 +2031,7 @@ field has a comment then that will be preserved (assuming field_comment is None). :param field_comment: If not None, add or replace the comment for - the field. Each string in the in the list will become one comment + the field. Each string in the list will become one comment line (inserted directly before the field name). Will appear in the same order as they do in the list. @@ -2052,7 +2106,7 @@ @overload def dump(self, - fd, # type: IO[bytes] + fd # type: IO[bytes] ): # type: (...) -> None pass @@ -2063,7 +2117,7 @@ pass def dump(self, - fd=None, # type: Optional[IO[bytes]] + fd=None # type: Optional[IO[bytes]] ): # type: (...) -> Optional[str] if fd is None: @@ -2082,7 +2136,7 @@ def __init__(self, kvpair_elements, # type: List[Deb822KeyValuePairElement] - kvpair_order, # type: OrderedSet + kvpair_order # type: OrderedSet ): # type: (...) -> None super().__init__() @@ -2126,8 +2180,13 @@ unpacked_ref_field, _, _ = _unpack_key(reference_field, raise_if_indexed=True) self._kvpair_order.order_after(unpacked_field, unpacked_ref_field) + # Overload to narrow the type to just str. + def __iter__(self): + # type: () -> Iterator[str] + return iter(str(k) for k in self._kvpair_order) + def iter_keys(self): - # type: () -> Iterable[ParagraphKey] + # type: () -> Iterable[str] yield from (str(k) for k in self._kvpair_order) def remove_kvpair_element(self, key): @@ -2146,7 +2205,7 @@ def get_kvpair_element(self, item, # type: ParagraphKey - use_get=False, # type: bool + use_get=False # type: bool ): # type: (...) -> Optional[Deb822KeyValuePairElement] item, _, _ = _unpack_key(item, raise_if_indexed=True) @@ -2359,8 +2418,8 @@ nodes, # type: List[KVPNode] key, # type: str index, # type: Optional[int] - name_token, # type: Optional[Deb822FieldNameToken] - use_get=False, # type: bool + name_token, # type: Optional[Deb822FieldNameToken] + use_get=False # type: bool ): # type: (...) -> Optional[KVPNode] if index is None: @@ -2386,7 +2445,7 @@ def get_kvpair_element(self, item, # type: ParagraphKey - use_get=False, # type: bool + use_get=False # type: bool ): # type: (...) -> Optional[Deb822KeyValuePairElement] key, index, name_token = _unpack_key(item) @@ -2404,7 +2463,7 @@ @staticmethod def _find_node_via_name_token( name_token, # type: Deb822FieldNameToken - elements, # type: Iterable[KVPNode] + elements # type: Iterable[KVPNode] ): # type: (...) -> Optional[KVPNode] # if we are given a name token, then it is non-ambiguous if we have exactly @@ -2618,7 +2677,7 @@ ... Package: libfoo-dev ... Depends: libfoo1 (= ${binary:Version}), ${shlib:Depends}, ${misc:Depends} ... '''.lstrip() - >>> deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + >>> deb822_file = parse_deb822_file(original.splitlines()) >>> para1 = Deb822ParagraphElement.new_empty_paragraph() >>> para1["Source"] = "foo" >>> para1["Build-Depends"] = "debhelper-compat (= 13)" @@ -2708,7 +2767,27 @@ if tail_element and not isinstance(tail_element, Deb822WhitespaceToken): self._token_and_elements.append(self._set_parent(Deb822WhitespaceToken('\n'))) self._token_and_elements.append(self._set_parent(paragraph)) - paragraph.parent_element = self + + def remove(self, paragraph): + # type: (Deb822ParagraphElement) -> None + if paragraph.parent_element is not self: + raise ValueError("Paragraph is part of a different file") + node = None + for node in self._token_and_elements.iter_nodes(): + if node.value is paragraph: + break + if node is None: + raise RuntimeError("unable to find paragraph") + previous_node = node.previous_node + next_node = node.next_node + self._token_and_elements.remove_node(node) + if next_node is None: + if previous_node and isinstance(previous_node.value, Deb822WhitespaceToken): + self._token_and_elements.remove_node(previous_node) + else: + if isinstance(next_node.value, Deb822WhitespaceToken): + self._token_and_elements.remove_node(next_node) + paragraph.parent_element = None def _set_parent(self, t): # type: (TE) -> TE @@ -2717,7 +2796,7 @@ @overload def dump(self, - fd, # type: IO[bytes] + fd # type: IO[bytes] ): # type: (...) -> None pass @@ -2728,7 +2807,7 @@ pass def dump(self, - fd=None, # type: Optional[IO[bytes]] + fd=None # type: Optional[IO[bytes]] ): # type: (...) -> Optional[str] if fd is None: @@ -2785,7 +2864,7 @@ return not isinstance(v, Deb822WhitespaceToken) or v.text != '\n' -def _build_value_line(token_stream, # type: Iterable[Union[TokenOrElement, Deb822CommentElement]] +def _build_value_line(token_stream # type: Iterable[Union[TokenOrElement, Deb822CommentElement]] ): # type: (...) -> Iterable[Union[TokenOrElement, Deb822ValueLineElement]] """Parser helper - consumes tokens part of a Deb822ValueEntryElement and turns them into one""" @@ -2824,7 +2903,13 @@ for token in buffered_stream: start_of_value_entry = False - if isinstance(token, Deb822CommentElement): + if isinstance(token, Deb822ValueContinuationToken): + continuation_line_token = token + start_of_value_entry = True + token = None + elif isinstance(token, Deb822FieldSeparatorToken): + start_of_value_entry = True + elif isinstance(token, Deb822CommentElement): next_token = buffered_stream.peek() # If the next token is a continuation line token, then this comment # belong to a value and we might as well just start the value @@ -2843,12 +2928,6 @@ next(buffered_stream, None) ) assert continuation_line_token is not None - elif isinstance(token, Deb822ValueContinuationToken): - continuation_line_token = token - start_of_value_entry = True - token = None - elif isinstance(token, Deb822FieldSeparatorToken): - start_of_value_entry = True if token is not None: yield token @@ -2889,7 +2968,9 @@ for token_or_element in buffered_stream: start_of_field = False comment_element = None - if isinstance(token_or_element, Deb822CommentElement): + if isinstance(token_or_element, Deb822FieldNameToken): + start_of_field = True + elif isinstance(token_or_element, Deb822CommentElement): comment_element = token_or_element next_token = buffered_stream.peek() start_of_field = isinstance(next_token, Deb822FieldNameToken) @@ -2899,29 +2980,24 @@ token_or_element = next(buffered_stream) except StopIteration: # pragma: no cover raise AssertionError - elif isinstance(token_or_element, Deb822FieldNameToken): - start_of_field = True if start_of_field: field_name = token_or_element - next_tokens = buffered_stream.peek_many(2) - if len(next_tokens) < 2: + separator = next(buffered_stream, None) + value_element = next(buffered_stream, None) + if separator is None or value_element is None: # Early EOF - should not be possible with how the tokenizer works # right now, but now it is future proof. if comment_element: yield comment_element error_elements = [field_name] - error_elements.extend(buffered_stream) + if separator is not None: + error_elements.append(separator) yield Deb822ErrorElement(error_elements) return - separator, value_element = next_tokens if isinstance(separator, Deb822FieldSeparatorToken) \ and isinstance(value_element, Deb822ValueElement): - # Consume the two tokens to align the stream - next(buffered_stream, None) - next(buffered_stream, None) - yield Deb822KeyValuePairElement(comment_element, cast('Deb822FieldNameToken', field_name), separator, @@ -2942,10 +3018,23 @@ yield token_or_element -def parse_deb822_file(sequence, # type: Iterable[Union[str, bytes]] +def _abort_on_error_tokens(sequence): + # type: (Iterable[TokenOrElement]) -> Iterable[TokenOrElement] + for token in sequence: + # We are always called while the sequence consists entirely of tokens + if isinstance(token, Deb822ErrorToken): + error_as_text = token.text.replace('\n', '\\n') + raise ValueError('Syntax or Parse error on the line: "{error_as_text}"'.format( + error_as_text=error_as_text + )) + yield token + + +def parse_deb822_file(sequence, # type: Union[Iterable[Union[str, bytes]], str] *, accept_files_with_error_tokens=False, # type: bool - accept_files_with_duplicated_fields=False # type: bool + accept_files_with_duplicated_fields=False, # type: bool + encoding='utf-8' # type: str ): # type: (...) -> Deb822FileElement """ @@ -2973,33 +3062,34 @@ Deb822ParagraphElement.configured_view). If False, then this method will raise a ValueError if any duplicated fields are seen inside any paragraph. + :param encoding: The encoding to use (this is here to support Deb822-like + APIs, new code should not use this parameter). """ + + if isinstance(sequence, (str, bytes)): + # Match the deb822 API. + sequence = sequence.splitlines(True) + # The order of operations are important here. As an example, # _build_value_line assumes that all comment tokens have been merged # into comment elements. Likewise, _build_field_and_value assumes # that value tokens (along with their comments) have been combined # into elements. - tokens = tokenize_deb822_file(sequence) # type: Iterable[TokenOrElement] + tokens = tokenize_deb822_file(sequence, encoding=encoding) # type: Iterable[TokenOrElement] + if not accept_files_with_error_tokens: + tokens = _abort_on_error_tokens(tokens) tokens = _combine_comment_tokens_into_elements(tokens) tokens = _build_value_line(tokens) tokens = _combine_vl_elements_into_value_elements(tokens) tokens = _build_field_with_value(tokens) tokens = _combine_kvp_elements_into_paragraphs(tokens) # Combine any free-floating error tokens into error elements. We do - # this last as it enable other parts of the parser to include error + # this last as it enables other parts of the parser to include error # tokens in their error elements if they discover something is wrong. tokens = _combine_error_tokens_into_elements(tokens) deb822_file = Deb822FileElement(LinkedList(tokens)) - if not accept_files_with_error_tokens: - error_element = deb822_file.find_first_error_element() - if error_element is not None: - error_as_text = error_element.convert_to_text().replace('\n', '\\n') - raise ValueError('Syntax or Parse error on the line: "{error_as_text}"'.format( - error_as_text=error_as_text - )) - if not accept_files_with_duplicated_fields: for no, paragraph in enumerate(deb822_file): if isinstance(paragraph, Deb822DuplicateFieldsParagraphElement): diff -Nru python-debian-0.1.44/lib/debian/_deb822_repro/tokens.py python-debian-0.1.46/lib/debian/_deb822_repro/tokens.py --- python-debian-0.1.44/lib/debian/_deb822_repro/tokens.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/_deb822_repro/tokens.py 2022-07-08 16:45:29.000000000 +0000 @@ -16,7 +16,6 @@ from debian._deb822_repro.parsing import Deb822Element -_RE_WHITESPACE_LINE = re.compile(r'^\s+$') # Consume whitespace and a single word. _RE_WHITESPACE_SEPARATED_WORD_LIST = re.compile(r''' (?P\s*) # Consume any whitespace before the word @@ -93,24 +92,21 @@ Deb822Token. """ - __slots__ = ('_text', '_hash', '_parent_element', '__weakref__') + __slots__ = ('_text', '_parent_element', '__weakref__') def __init__(self, text): # type: (str) -> None if text == '': # pragma: no cover raise ValueError("Tokens must have content") self._text = text # type: str - self._hash = None # type: Optional[int] self._parent_element = None # type: Optional[ReferenceType['Deb822Element']] self._verify_token_text() def __repr__(self): # type: () -> str - if self._text != "": - return "{clsname}('{text}')".format(clsname=self.__class__.__name__, - text=self._text.replace('\n', '\\n') - ) - return self.__class__.__name__ + return "{clsname}('{text}')".format(clsname=self.__class__.__name__, + text=self._text.replace('\n', '\\n') + ) def _verify_token_text(self): # type: () -> None @@ -252,7 +248,8 @@ __slots__ = () - def __init__(self) -> None: + def __init__(self): + # type: () -> None super().__init__(':') @@ -261,7 +258,8 @@ __slots__ = () - def __init__(self) -> None: + def __init__(self): + # type: () -> None super().__init__(',') @@ -270,7 +268,8 @@ __slots__ = () - def __init__(self) -> None: + def __init__(self): + # type: () -> None super().__init__('|') @@ -291,52 +290,42 @@ __slots__ = () -def tokenize_deb822_file(sequence: Iterable[Union[str, bytes]]) -> Iterable[Deb822Token]: - # type(Iterable[Union[str, bytes]]) -> Iterable[Deb822Token] +def tokenize_deb822_file(sequence, encoding='utf-8'): + # type: (Iterable[Union[str, bytes]], str) -> Iterable[Deb822Token] """Tokenize a deb822 file :param sequence: An iterable of lines (a file open for reading will do) + :param encoding: The encoding to use (this is here to support Deb822-like + APIs, new code should not use this parameter). """ current_field_name = None field_name_cache = {} # type: Dict[str, _strI] - def _as_str(s: Iterable[Union[str, bytes]]) -> Iterable[str]: + def _normalize_input(s): + # type: (Iterable[Union[str, bytes]]) -> Iterable[str] for x in s: if isinstance(x, bytes): - x = x.decode('utf-8') + x = x.decode(encoding) + if not x.endswith("\n"): + # We always end on a newline because it makes a lot of code simpler. The pain + # points relates to mutations that add content after the last field. Sadly, these + # mutations can happen via adding fields, reordering fields, etc. and are too hard + # to track to make it worth it to support the special case that makes up missing + # a newline at the end of the file. + x += "\n" yield x - text_stream = BufferingIterator(_as_str(sequence)) # type: BufferingIterator[str] - auto_correct_newlines = False - first_line = text_stream.peek() - if first_line is not None and not first_line.endswith("\n"): - # Special-case: Single line files count as "last line without a newline" rather than - # auto-correction. - auto_correct_newlines = text_stream.peek_at(2) is not None - - for no, line in enumerate(text_stream, start=1): - if auto_correct_newlines: - if line.endswith("\n"): - raise ValueError("Input is inconsistent with its line endings! Lines must " - "consistently be *with* or *without* line endings") - line += "\n" - - if not line.endswith("\n"): - # We expect newlines at the end of each line except the last. - if text_stream.peek() is not None: - raise ValueError("Invalid line iterator: Line " + str(no) + " did not end on a" - " newline and it is not the last line in the stream!") - if line == '': - raise ValueError("Line " + str(no) + " was completely empty. The tokenizer expects" - " whitespace (including newlines) to be present") - if _RE_WHITESPACE_LINE.match(line): + text_stream = BufferingIterator(_normalize_input(sequence)) # type: BufferingIterator[str] + + for line in text_stream: + if line.isspace(): if current_field_name: # Blank lines terminate fields current_field_name = None # If there are multiple whitespace-only lines, we combine them # into one token. - r = list(text_stream.takewhile(lambda x: _RE_WHITESPACE_LINE.match(x) is not None)) + r = list(text_stream.takewhile(str.isspace)) if r: line += "".join(r) @@ -354,17 +343,11 @@ # We emit a separate whitespace token for the newline as it makes some # things easier later (see _build_value_line) leading = sys.intern(line[0]) - if line.endswith('\n'): - line = line[1:-1] - emit_newline_token = True - else: - line = line[1:] - emit_newline_token = False - + # Pull out the leading space and newline + line = line[1:-1] yield Deb822ValueContinuationToken(leading) yield Deb822ValueToken(line) - if emit_newline_token: - yield Deb822NewlineAfterValueToken() + yield Deb822NewlineAfterValueToken() else: yield Deb822ErrorToken(line) continue @@ -377,7 +360,6 @@ (field_name, _, space_before, value, space_after) = field_line_match.groups() current_field_name = field_name_cache.get(field_name) - emit_newline_token = False if value is None or value == '': # If there is no value, then merge the two space elements into space_after @@ -388,8 +370,7 @@ if space_after: # We emit a separate whitespace token for the newline as it makes some # things easier later (see _build_value_line) - emit_newline_token = space_after.endswith('\n') - if emit_newline_token: + if space_after.endswith('\n'): space_after = space_after[:-1] if current_field_name is None: @@ -410,8 +391,7 @@ yield Deb822ValueToken(value) if space_after: yield Deb822WhitespaceToken(sys.intern(space_after)) - if emit_newline_token: - yield Deb822NewlineAfterValueToken() + yield Deb822NewlineAfterValueToken() else: yield Deb822ErrorToken(line) @@ -421,8 +401,8 @@ def impl(v): # type: (str) -> Iterable[Deb822Token] first_line = True - for line in v.splitlines(keepends=True): - assert not _RE_WHITESPACE_LINE.match(v) + for no, line in enumerate(v.splitlines(keepends=True)): + assert not v.isspace() or no == 0 if line.startswith("#"): yield Deb822CommentToken(line) continue diff -Nru python-debian-0.1.44/lib/debian/_deb822_repro/types.py python-debian-0.1.46/lib/debian/_deb822_repro/types.py --- python-debian-0.1.44/lib/debian/_deb822_repro/types.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/_deb822_repro/types.py 2022-07-08 16:45:29.000000000 +0000 @@ -10,46 +10,20 @@ TokenOrElement = Union['Deb822Element', 'Deb822Token'] TE = TypeVar('TE', bound=TokenOrElement) - TE.__doc__ = """ - Generic "Token or Element" type - """ # Used as a resulting element for "mapping" functions that map TE -> R (see _combine_parts) R = TypeVar('R', bound='Deb822Element') - R.__doc__ = """ - For internal usage in _deb822_repro - """ VE = TypeVar('VE', bound='Deb822Element') - VE.__doc__ = """ - Value type/element in a list interpretation of a field value - """ ST = TypeVar('ST', bound='Deb822Token') - ST.__doc__ = """ - Separator type/token in a list interpretation of a field value - """ # Internal type for part of the paragraph key. Used to facility _unpack_key. ParagraphKeyBase = Union['Deb822FieldNameToken', str] - ParagraphKeyBase.__doc__ = """ - For internal usage in _deb822_repro - """ ParagraphKey = Union[ParagraphKeyBase, Tuple[str, int]] - ParagraphKey.__doc__ = """ - Anything accepted as a key for a paragraph field lookup. The simple case being - a str. Alternative variants are mostly interesting for paragraphs with repeated - fields (to enable unambiguous lookups) - """ Commentish = Union[List[str], 'Deb822CommentElement'] - Commentish.__doc__ = """ - Anything accepted as input for a Comment. The simple case is the list - of string (each element being a line of comment). The alternative format is - there for enable reuse of an existing element (e.g. to avoid "unpacking" - only to "re-pack" an existing comment element). - """ FormatterCallback = Callable[[ str, @@ -58,11 +32,42 @@ ], Iterator[Union['FormatterContentToken', str]] ] - FormatterCallback.__doc__ = """\ - Formatter callback used with the round-trip safe parser - - See debian._repro_deb822.formatter.format_field for details - """ + try: + # Set __doc__ attributes if possible + TE.__doc__ = """ + Generic "Token or Element" type + """ + R.__doc__ = """ + For internal usage in _deb822_repro + """ + VE.__doc__ = """ + Value type/element in a list interpretation of a field value + """ + ST.__doc__ = """ + Separator type/token in a list interpretation of a field value + """ + ParagraphKeyBase.__doc__ = """ + For internal usage in _deb822_repro + """ + ParagraphKey.__doc__ = """ + Anything accepted as a key for a paragraph field lookup. The simple case being + a str. Alternative variants are mostly interesting for paragraphs with repeated + fields (to enable unambiguous lookups) + """ + Commentish.__doc__ = """ + Anything accepted as input for a Comment. The simple case is the list + of string (each element being a line of comment). The alternative format is + there for enable reuse of an existing element (e.g. to avoid "unpacking" + only to "re-pack" an existing comment element). + """ + FormatterCallback.__doc__ = """\ + Formatter callback used with the round-trip safe parser + + See debian._repro_deb822.formatter.format_field for details + """ + except AttributeError: + # Python 3.5 does not allow update to the __doc__ attribute - ignore that + pass except ImportError: pass diff -Nru python-debian-0.1.44/lib/debian/_deb822_repro/_util.py python-debian-0.1.46/lib/debian/_deb822_repro/_util.py --- python-debian-0.1.44/lib/debian/_deb822_repro/_util.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/_deb822_repro/_util.py 2022-07-08 16:45:29.000000000 +0000 @@ -1,13 +1,15 @@ import collections import collections.abc import logging +import sys import textwrap +from abc import ABC try: from typing import ( Optional, Union, Iterable, Callable, TYPE_CHECKING, Iterator, - Type, cast, List, - ) + Type, cast, List, Generic, +) from debian._util import T from debian._deb822_repro.types import TE, R, TokenOrElement @@ -28,7 +30,7 @@ def print_ast(ast_tree, # type: Union[Iterable[TokenOrElement], 'Deb822Element'] *, end_marker_after=5, # type: Optional[int] - output_function=None, # type: Optional[Callable[[str], None]] + output_function=None # type: Optional[Callable[[str], None]] ): # type: (...) -> None """Debugging aid, which can dump a Deb822Element or a list of tokens/elements @@ -81,7 +83,7 @@ def combine_into_replacement(source_class, # type: Type[TE] replacement_class, # type: Type[R] *, - constructor=None, # type: Optional[Callable[[List[TE]], R]] + constructor=None # type: Optional[Callable[[List[TE]], R]] ): # type: (...) -> _combine_parts_ret_type[TE, R] """Combines runs of one type into another type @@ -114,9 +116,19 @@ return _impl -class BufferingIterator(collections.abc.Iterator[T]): +if sys.version_info >= (3, 8) or TYPE_CHECKING: + _bufferingIterator_Base = collections.abc.Iterator[T] +else: + # Python 3.5 - 3.7 compat - we are not allowed to subscript the abc.Iterator + # - use this little hack to work around it + class _bufferingIterator_Base(collections.abc.Iterator, Generic[T], ABC): + pass - def __init__(self, stream: Iterable[T]) -> None: + +class BufferingIterator(_bufferingIterator_Base[T], Generic[T]): + + def __init__(self, stream): + # type: (Iterable[T]) -> None self._stream = iter(stream) # type: Iterator[T] self._buffer = collections.deque() # type: collections.deque[T] self._expired = False # type: bool @@ -161,7 +173,7 @@ def peek_find(self, predicate, # type: Callable[[T], bool] - limit=None, # type: Optional[int] + limit=None # type: Optional[int] ): # type: (...) -> Optional[int] buffer = self._buffer @@ -217,7 +229,7 @@ def len_check_iterator(content, # type: str stream, # type: Iterable[TE] - content_len=None, # type: Optional[int] + content_len=None # type: Optional[int] ): # type: (...) -> Iterable[TE] """Flatten a parser's output into tokens and verify it covers the entire line/text""" diff -Nru python-debian-0.1.44/lib/debian/debian_support.py python-debian-0.1.46/lib/debian/debian_support.py --- python-debian-0.1.44/lib/debian/debian_support.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/debian_support.py 2022-07-08 16:45:29.000000000 +0000 @@ -49,8 +49,13 @@ try: import apt_pkg - apt_pkg.init() - _have_apt_pkg = True + try: + apt_pkg.init() + _have_apt_pkg = True + except apt_pkg.Error: + # If dpkg (e.g., tupledata) is missing, we can import apt_pkg but .init() + # will raise an exception + _have_apt_pkg = False except ImportError: _have_apt_pkg = False @@ -84,6 +89,12 @@ " incompatibilities") +# Re-exports +import debian._arch_table + +DpkgArchTable = debian._arch_table.DpkgArchTable + + class ParseError(Exception): """An exception which is used to signal a parse failure. diff -Nru python-debian-0.1.44/lib/debian/substvars.py python-debian-0.1.46/lib/debian/substvars.py --- python-debian-0.1.44/lib/debian/substvars.py 1970-01-01 00:00:00.000000000 +0000 +++ python-debian-0.1.46/lib/debian/substvars.py 2022-07-08 16:45:29.000000000 +0000 @@ -0,0 +1,399 @@ +""" Facilities for reading and writing Debian substvars files + +The aim of this module is to provide programmatic access to Debian substvars +files to query and manipulate them. The format for the changelog is defined in +`deb-substvars(5) +`_ + +Overview +======== + +The most common use-case for substvars is for package helpers to add or update +a substvars (e.g., to add a dependency). This would look something like: + + >>> from debian.substvars import Substvars + >>> from tempfile import TemporaryDirectory + >>> import os + >>> # Using a tmp dir for the sake of doctests + >>> with TemporaryDirectory() as debian_dir: + ... filename = os.path.join(debian_dir, "foo.substvars") + ... with Substvars.load_from_path(filename, missing_ok=True) as svars: + ... svars.add_dependency("misc:Depends", "bar (>= 1.0)") + +By default, the module creates new substvars as "mandatory" substvars (that +triggers a warning by dpkg-gecontrol if not used. However, it does also +support the "optional" substvars introduced in dpkg 1.21.8. See +`Substvars.as_substvars` for an example of how to use the "optional" +substvars. + + +The :class:`Substvars` class is the key class within this module. + +Substvars Classes +----------------- +""" + + +import contextlib +import errno +import re +import sys +import typing +from abc import ABC +from collections import OrderedDict +from collections.abc import MutableMapping +from types import TracebackType +from typing import Dict, Set, Optional, Union, Iterator, IO, Iterable, TYPE_CHECKING, Type + +try: + from os import PathLike + AnyPath = Union[PathLike, str, bytes] +except ImportError: + pass + +T = typing.TypeVar('T') + +_SUBSTVAR_PATTERN = re.compile( + r"^(?P\w[-:\dA-Za-z]*)(?P[?]?=)(?P.*)$" +) + + +class Substvar: + + __slots__ = ['_assignment_operator', '_value'] + + def __init__(self, initial_value="", # type: str + assignment_operator='=', # type: str + ): + # type: (...) -> None + + # We have 2 values for _value: + # 1) string: The variable is set to a fixed string. This variant is + # round-trip safe. + # 2) set: The variable is dependency-like field. This variant is *NOT* + # round-trip safe. + # + # When reading substvars from files, we always use variant 1) and then + # lazily convert to 2) when necessary. This choice makes the substvars + # round-trip safe by default until someone messes with a substvar. + self._value = initial_value # type: Union[str, Set[str]] + self.assignment_operator = assignment_operator # type: str + + @property + def assignment_operator(self): + # type: () -> str + return self._assignment_operator + + @assignment_operator.setter + def assignment_operator(self, new_operator): + # type: (str) -> None + if new_operator not in {'=', '?='}: + raise ValueError('Operator must be one of: "=", or "?=" - got: ' + new_operator) + self._assignment_operator = new_operator + + def add_dependency(self, dependency_clause): + # type: (str) -> None + if self._value == "": + self._value = {dependency_clause} + return + if isinstance(self._value, str): + # Convert to dependency format + self._value = {v.strip() for v in self._value.split(',')} + self._value.add(dependency_clause) + + def resolve(self): + # type: () -> str + if isinstance(self._value, set): + return ", ".join(sorted(self._value)) + return self._value + + def __eq__(self, other): + # type: (object) -> bool + if other is None or not isinstance(other, Substvar): + return False + if self.assignment_operator != other.assignment_operator: + return False + return self.resolve() == other.resolve() + + +if sys.version_info >= (3, 8) or TYPE_CHECKING: + class _Substvars_Base(contextlib.AbstractContextManager[T], MutableMapping[str, str], ABC): + pass +else: + # Python 3.5 - 3.7 compat - we are not allowed to subscript the abc.MutableMapping + # - use this little hack to work around it + # Note that Python 3.5 is so old that it does not have AbstractContextManager, + # so we re-implement it here as well. + class _Substvars_Base(typing.Generic[T], MutableMapping, ABC): + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return None + + +class Substvars(_Substvars_Base['Substvars']): + """Substvars is a dict-like object containing known substvars for a given package. + + >>> substvars = Substvars() + >>> substvars['foo'] = 'bar, golf' + >>> substvars['foo'] + 'bar, golf' + >>> substvars.add_dependency('foo', 'dpkg (>= 1.20.0)') + >>> substvars['foo'] + 'bar, dpkg (>= 1.20.0), golf' + >>> 'foo' in substvars + True + >>> sorted(substvars) + ['foo'] + >>> del substvars['foo'] + >>> substvars['foo'] + Traceback (most recent call last): + ... + KeyError: 'foo' + >>> substvars.get('foo') + >>> # None + >>> substvars['foo'] = "" + >>> substvars['foo'] + '' + + The Substvars object also provide methods for serializing and deserializing + the substvars into and from the format used by dpkg-gencontrol. + + The Substvars object can be used as a context manager, which causes the substvars + to be saved when the context manager exits successfully (i.e., no exceptions are raised). + """ + + __slots__ = ['_vars_dict', '_substvars_path'] + + def __init__(self): + # type: () -> None + self._vars_dict = OrderedDict() # type: Dict[str, Substvar] + self._substvars_path = None # type: Optional[AnyPath] + + @classmethod + def load_from_path(cls, substvars_path, missing_ok=False): + # type: (AnyPath, bool) -> Substvars + """Shorthand for initializing a Substvars from a file + + The return substvars will have `substvars_path` set to the provided path enabling + `save()` to work out of the box. This also makes it easy to combine this with the + context manager interface to automatically save the file again. + + >>> import os + >>> from tempfile import TemporaryDirectory + >>> with TemporaryDirectory() as tmpdir: + ... filename = os.path.join(tmpdir, "foo.substvars") + ... # Obviously, this does not exist + ... print("Exists before: " + str(os.path.exists(filename))) + ... with Substvars.load_from_path(filename, missing_ok=True) as svars: + ... svars.add_dependency("misc:Depends", "bar (>= 1.0)") + ... print("Exists after: " + str(os.path.exists(filename))) + Exists before: False + Exists after: True + + :param substvars_path: The path to load from + :param missing_ok: If True, then the path does not have to exist (i.e. + FileNotFoundError causes an empty Substvars object to be returned). Combined + with the context manager, this is useful for packaging helpers that want to + append / update to the existing if it exists or create it if it does not exist. + """ + substvars = cls() + try: + with open(substvars_path, 'r', encoding='utf-8') as fd: + substvars.read_substvars(fd) + except OSError as e: + if e.errno != errno.ENOENT or not missing_ok: + raise + substvars.substvars_path = substvars_path + return substvars + + @property + def _vars(self): + # type: () -> Dict[str, Substvar] + # Indirection to support subclasses that want to provide lazy loading or other "fun stuff" + return self._vars_dict + + @_vars.setter + def _vars(self, vars_dict): + # type: (Dict[str, Substvar]) -> None + # Indirection to support subclasses that want to provide lazy loading or other "fun stuff" + self._vars_dict = vars_dict + + @property + def substvars_path(self): + # type: () -> Optional[AnyPath] + return self._substvars_path + + @substvars_path.setter + def substvars_path(self, new_path): + # type: (Optional[AnyPath]) -> None + self._substvars_path = new_path + + def add_dependency(self, substvar, dependency_clause): + # type: (str, str) -> None + """Add a dependency clause to a given substvar + + >>> substvars = Substvars() + >>> # add_dependency automatically creates variables + >>> 'misc:Recommends' not in substvars + True + >>> substvars.add_dependency('misc:Recommends', "foo (>= 1.0)") + >>> substvars['misc:Recommends'] + 'foo (>= 1.0)' + >>> # It can be appended to other variables + >>> substvars['foo'] = 'bar, golf' + >>> substvars.add_dependency('foo', 'dpkg (>= 1.20.0)') + >>> substvars['foo'] + 'bar, dpkg (>= 1.20.0), golf' + >>> # Exact duplicates are ignored + >>> substvars.add_dependency('foo', 'dpkg (>= 1.20.0)') + >>> substvars['foo'] + 'bar, dpkg (>= 1.20.0), golf' + + """ + try: + variable = self._vars[substvar] + except KeyError: + variable = Substvar() + self._vars[substvar] = variable + variable.add_dependency(dependency_clause) + + def __exit__(self, + exc_type, # type: Optional[Type[BaseException]] + exc_val, # type: Optional[BaseException] + exc_tb, # type: Optional[TracebackType] + ): + # type: (...) -> Optional[bool] + if exc_type is None: + self.save() + return super().__exit__(exc_type, exc_val, exc_tb) + + def __iter__(self): + # type: () -> Iterator[str] + return iter(self._vars) + + def __len__(self) -> int: + return len(self._vars_dict) + + def __contains__(self, item): + # type: (object) -> bool + return item in self._vars + + def __getitem__(self, key): + # type: (str) -> str + return self._vars[key].resolve() + + def __delitem__(self, key): + # type: (str) -> None + del self._vars[key] + + def __setitem__(self, key, value): + # type: (str, str) -> None + self._vars[key] = Substvar(value) + + @property + def as_substvar(self): + # type: () -> MutableMapping[str, Substvar] + """Provides a mapping to the Substvars object for more advanced operations + + Treating a substvars file mostly as a "str -> str" mapping is sufficient for many cases. + But when full control over the substvars (like fiddling with the assignment operator) is + needed this attribute is useful. + + >>> content = ''' + ... # Some comment (which is allowed but no one uses them - also, they are not preserved) + ... shlib:Depends=foo (>= 1.0), libbar2 (>= 2.1-3~) + ... random:substvar?=With the new assignment operator from dpkg 1.21.8 + ... ''' + >>> substvars = Substvars() + >>> substvars.read_substvars(content.splitlines()) + >>> substvars.as_substvar["shlib:Depends"].assignment_operator + '=' + >>> substvars.as_substvar["random:substvar"].assignment_operator + '?=' + >>> # Mutation is also possible + >>> substvars.as_substvar["shlib:Depends"].assignment_operator = '?=' + >>> print(substvars.dump(), end="") + shlib:Depends?=foo (>= 1.0), libbar2 (>= 2.1-3~) + random:substvar?=With the new assignment operator from dpkg 1.21.8 + """ + # This is an indirection of `_vars` to avoid exposing the `_vars` setter + return self._vars + + def __eq__(self, other): + # type: (object) -> bool + if other is None or not isinstance(other, Substvars): + return False + return self._vars == other._vars + + def dump(self): + # type: () -> str + """Debug aid that generates a string representation of the content + + For persisting the contents, please consider `save()` or `write_substvars`. + """ + return "".join("{}{}{}\n".format(k, v.assignment_operator, v.resolve()) + for k, v in self._vars.items() + ) + + def save(self): + # type: () -> None + """Save the substvars file + + Replace the path denoted by the `substvars_path` attribute with the + in-memory version of the substvars. Note that the `substvars_path` + property must be not None for this method to work. + """ + if self._substvars_path is None: + raise TypeError("The substvar does not have a substvars_path: Please" + " set substvars_path first or use write_substvars") + + with open(self._substvars_path, 'w', encoding='utf-8') as fd: + return self.write_substvars(fd) + + def write_substvars(self, fileobj): + # type: (IO[str]) -> None + """Write a copy of the substvars to an open text file + + :param fileobj: The open file (should open in text mode using the UTF-8 encoding) + """ + fileobj.writelines("{}{}{}\n".format(k, v.assignment_operator, v.resolve()) + for k, v in self._vars.items() + ) + + def read_substvars(self, fileobj): + # type: (Iterable[str]) -> None + """Read substvars from an open text file in the format supported by dpkg-gencontrol + + On success, all existing variables will be discarded and only variables + from the file will be present after this method completes. In case of + any IO related errors, the object retains its state prior to the call + of this method. + + >>> content = ''' + ... # Some comment (which is allowed but no one uses them - also, they are not preserved) + ... shlib:Depends=foo (>= 1.0), libbar2 (>= 2.1-3~) + ... random:substvar?=With the new assignment operator from dpkg 1.21.8 + ... ''' + >>> substvars = Substvars() + >>> substvars.read_substvars(content.splitlines()) + >>> substvars["shlib:Depends"] + 'foo (>= 1.0), libbar2 (>= 2.1-3~)' + >>> substvars["random:substvar"] + 'With the new assignment operator from dpkg 1.21.8' + + :param fileobj: An open file (in text mode using the UTF-8 encoding) or an + iterable of str that provides line by line content. + """ + vars_dict = OrderedDict() + for line in fileobj: + if line.strip() == '' or line[0] == '#': + continue + m = _SUBSTVAR_PATTERN.match(line.rstrip("\r\n")) + if not m: + continue + varname, assignment_operator, value = m.groups() + vars_dict[varname] = Substvar(value, assignment_operator=assignment_operator) + self._vars = vars_dict diff -Nru python-debian-0.1.44/lib/debian/tests/stubbed_arch_table.py python-debian-0.1.46/lib/debian/tests/stubbed_arch_table.py --- python-debian-0.1.44/lib/debian/tests/stubbed_arch_table.py 1970-01-01 00:00:00.000000000 +0000 +++ python-debian-0.1.46/lib/debian/tests/stubbed_arch_table.py 2022-07-08 16:45:29.000000000 +0000 @@ -0,0 +1,49 @@ +from io import StringIO +try: + from os import PathLike + from typing import Union +except ImportError: + pass + +from debian._arch_table import DpkgArchTable + +stubbed_cpu_table_data = """\ +# Version=1.0 +# +# This file contains the table of known CPU names. +# +# [...] +# +# +i386 i686 (i[34567]86|pentium) 32 little +amd64 x86_64 (amd64|x86_64) 64 little +arm arm arm.* 32 little +arm64 aarch64 aarch64 64 little +""" + +stubbed_tuple_table_data = """\ +# Version=1.0 +# +# [...] +# +# Supported variables: +# +# +eabihf-gnu-linux-arm armhf +eabi-gnu-linux-arm armel +x32-gnu-linux-amd64 x32 +base-gnu-linux- +eabihf-gnu-kfreebsd-arm kfreebsd-armhf +base-gnu-kfreebsd- kfreebsd- +""" + + +class StubbedDpkgArchTable(DpkgArchTable): + + @classmethod + def load_arch_table(cls, path="/usr/share/dpkg"): + # type: (Union[str, PathLike[str]]) -> DpkgArchTable + cpu_table = StringIO(stubbed_cpu_table_data) + tuple_table = StringIO(stubbed_tuple_table_data) + return cls._from_file(tuple_table, cpu_table) + diff -Nru python-debian-0.1.44/lib/debian/tests/test_arch_table.py python-debian-0.1.46/lib/debian/tests/test_arch_table.py --- python-debian-0.1.44/lib/debian/tests/test_arch_table.py 1970-01-01 00:00:00.000000000 +0000 +++ python-debian-0.1.46/lib/debian/tests/test_arch_table.py 2022-07-08 16:45:29.000000000 +0000 @@ -0,0 +1,123 @@ +#! /usr/bin/python3 +## vim: fileencoding=utf-8 + +# Copyright (C) 2022 Niels Thykier +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +import os.path +import unittest +from unittest import skipIf + +from debian.debian_support import DpkgArchTable +from .stubbed_arch_table import StubbedDpkgArchTable + + +if os.path.isfile("/usr/share/dpkg/tupletable"): + HAS_REAL_DATA = True +else: + HAS_REAL_DATA = False + + +class TestDpkgArchTable(unittest.TestCase): + + def test_matches_architecture(self): + # type: () -> None + arch_table = StubbedDpkgArchTable.load_arch_table() + self.assertTrue(arch_table.matches_architecture("amd64", "linux-any")) + self.assertTrue(arch_table.matches_architecture("i386", "linux-any")) + self.assertTrue(arch_table.matches_architecture("amd64", "amd64")) + + self.assertFalse(arch_table.matches_architecture("i386", "amd64")) + self.assertFalse(arch_table.matches_architecture("all", "amd64")) + + self.assertTrue(arch_table.matches_architecture("all", "all")) + + # i386 is the short form of linux-i386. Therefore, it does not match kfreebsd-i386 + self.assertFalse(arch_table.matches_architecture("i386", "kfreebsd-i386")) + + # Note that "armel" and "armhf" are "arm" CPUs, so it is matched by "any-arm" + # (similar holds for some other architecture <-> CPU name combinations) + for n in ['armel', 'armhf']: + self.assertTrue(arch_table.matches_architecture(n, 'any-arm')) + # Since "armel" is not a valid CPU name, this returns False (the correct would be + # any-arm as noted above) + self.assertFalse(arch_table.matches_architecture("armel", "any-armel")) + + # Wildcards used as architecture always fail (except for special cases noted in the + # compatibility notes below) + self.assertFalse(arch_table.matches_architecture("any-i386", "i386")) + + # any-i386 is not a subset of linux-any (they only have i386/linux-i386 as overlap) + self.assertFalse(arch_table.matches_architecture("any-i386", "linux-any")) + + # Compatibility with dpkg - if alias is `any` then it always returns True + # even if the input otherwise would not make sense. + self.assertTrue(arch_table.matches_architecture("any-unknown", "any")) + # Another side effect of the dpkg compatibility + self.assertTrue(arch_table.matches_architecture("all", "any")) + + # STUB VERIFICATION: This would return True if we used real data. But we are supposed to + # use the stub which does not have data for this architecture. + # (If this fails because you added the architecture to the stub, then replace it with + # another architecture, so the verification still works) + self.assertFalse(arch_table.matches_architecture('mipsel', 'any-mipsel')) + + def test_arch_equals(self): + # type: () -> None + arch_table = StubbedDpkgArchTable.load_arch_table() + self.assertTrue(arch_table.architecture_equals("linux-amd64", "amd64")) + self.assertFalse(arch_table.architecture_equals("amd64", "linux-i386")) + self.assertFalse(arch_table.architecture_equals("i386", "linux-amd64")) + self.assertTrue(arch_table.architecture_equals("amd64", "amd64")) + self.assertFalse(arch_table.architecture_equals("i386", "amd64")) + + # Compatibility with dpkg: if the parameters are equal, then it always return True + self.assertTrue(arch_table.architecture_equals("unknown", "unknown")) + + def test_architecture_is_concerned(self): + # type: () -> None + arch_table = StubbedDpkgArchTable.load_arch_table() + self.assertTrue(arch_table.architecture_is_concerned("linux-amd64", ["amd64", "i386"])) + self.assertFalse(arch_table.architecture_is_concerned("amd64", ["!amd64", "!i386"])) + # This is False because the "!amd64" is matched first. + self.assertFalse(arch_table.architecture_is_concerned( + "linux-amd64", + ["!linux-amd64", "linux-any"], + allow_mixing_positive_and_negative=True + )) + # This is True because the "linux-any" is matched first. + self.assertTrue(arch_table.architecture_is_concerned( + "linux-amd64", + ["linux-any", "!linux-amd64"], + allow_mixing_positive_and_negative=True + )) + + def test_is_wildcard(self): + # type: () -> None + arch_table = StubbedDpkgArchTable.load_arch_table() + self.assertTrue(arch_table.is_wildcard("linux-any")) + self.assertFalse(arch_table.is_wildcard("amd64")) + self.assertFalse(arch_table.is_wildcard("unknown")) + # Compatibility with the dpkg version of the function. + self.assertTrue(arch_table.is_wildcard("unknown-any")) + + @skipIf(not HAS_REAL_DATA, "Missing real data") + def test_has_real_data(self): + # type: () -> None + arch_table = DpkgArchTable.load_arch_table() + # The tests here rely on the production data, so we can use mips (which is not present in + # our stubbed data). + + self.assertTrue(arch_table.matches_architecture('mipsel', 'any-mipsel')) diff -Nru python-debian-0.1.44/lib/debian/tests/test_copyright.py python-debian-0.1.46/lib/debian/tests/test_copyright.py --- python-debian-0.1.44/lib/debian/tests/test_copyright.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/tests/test_copyright.py 2022-07-08 16:45:29.000000000 +0000 @@ -23,6 +23,7 @@ from debian import copyright from debian import deb822 +from debian._deb822_repro import parse_deb822_file, Deb822ParagraphElement try: @@ -134,6 +135,131 @@ FORMAT = 'https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/' +class RestrictedWrapperTest(unittest.TestCase): + class Wrapper(copyright._RestrictedWrapper): + restricted_field = deb822.RestrictedField('Restricted-Field') + required_field = deb822.RestrictedField('Required-Field', allow_none=False) + space_separated = deb822.RestrictedField( + 'Space-Separated', + from_str=lambda s: tuple((s or '').split()), + to_str=lambda seq: ' '.join(_no_space(s) for s in seq) or None) + + def test_unrestricted_get_and_set(self): + # type: () -> None + data = Deb822ParagraphElement.new_empty_paragraph() + data['Foo'] = 'bar' + + wrapper = self.Wrapper(data) + self.assertEqual('bar', wrapper['Foo']) + wrapper['foo'] = 'baz' + self.assertEqual('baz', wrapper['Foo']) + self.assertEqual('baz', wrapper['foo']) + + multiline = 'First line\n Another line' + wrapper['X-Foo-Bar'] = multiline + self.assertEqual(multiline, wrapper['X-Foo-Bar']) + self.assertEqual(multiline, wrapper['x-foo-bar']) + + expected_data = Deb822ParagraphElement.new_empty_paragraph() + expected_data['Foo'] = 'baz' + expected_data['X-Foo-Bar'] = multiline + self.assertEqual(expected_data.keys(), data.keys()) + self.assertEqual(expected_data, data) + + def test_trivially_restricted_get_and_set(self): # type: ignore + # mypy can't cope with the metaprogramming here + data = Deb822ParagraphElement.new_empty_paragraph() + data['Required-Field'] = 'some value' + + wrapper = self.Wrapper(data) + self.assertEqual('some value', wrapper.required_field) + self.assertEqual('some value', wrapper['Required-Field']) + self.assertEqual('some value', wrapper['required-field']) + self.assertIsNone(wrapper.restricted_field) + + with self.assertRaises(deb822.RestrictedFieldError): + wrapper['Required-Field'] = 'foo' + with self.assertRaises(deb822.RestrictedFieldError): + wrapper['required-field'] = 'foo' + with self.assertRaises(deb822.RestrictedFieldError): + wrapper['Restricted-Field'] = 'foo' + with self.assertRaises(deb822.RestrictedFieldError): + wrapper['Restricted-field'] = 'foo' + + with self.assertRaises(deb822.RestrictedFieldError): + del wrapper['Required-Field'] + with self.assertRaises(deb822.RestrictedFieldError): + del wrapper['required-field'] + with self.assertRaises(deb822.RestrictedFieldError): + del wrapper['Restricted-Field'] + with self.assertRaises(deb822.RestrictedFieldError): + del wrapper['restricted-field'] + + with self.assertRaises(TypeError): + wrapper.required_field = None # type: ignore + + wrapper.restricted_field = 'special value' # type: ignore + self.assertEqual('special value', data['Restricted-Field']) + wrapper.restricted_field = None # type: ignore + self.assertFalse('Restricted-Field' in data) + self.assertIsNone(wrapper.restricted_field) + + wrapper.required_field = 'another value' # type: ignore + self.assertEqual('another value', data['Required-Field']) + + def test_set_already_none_to_none(self): # type: ignore + # mypy can't cope with the metaprogramming here + data = Deb822ParagraphElement.new_empty_paragraph() + wrapper = self.Wrapper(data) + wrapper.restricted_field = 'Foo' # type: ignore + wrapper.restricted_field = None # type: ignore + self.assertFalse('restricted-field' in data) + wrapper.restricted_field = None # type: ignore + self.assertFalse('restricted-field' in data) + + def test_processed_get_and_set(self): # type: ignore + # mypy can't cope with the metaprogramming here + data = Deb822ParagraphElement.new_empty_paragraph() + data['Space-Separated'] = 'foo bar baz' + + wrapper = self.Wrapper(data) + self.assertEqual(('foo', 'bar', 'baz'), wrapper.space_separated) + wrapper.space_separated = ['bar', 'baz', 'quux'] # type: ignore + self.assertEqual('bar baz quux', data['space-separated']) + self.assertEqual('bar baz quux', wrapper['space-separated']) + self.assertEqual(('bar', 'baz', 'quux'), wrapper.space_separated) + + with self.assertRaises(ValueError) as cm: + wrapper.space_separated = ('foo', 'bar baz') # type: ignore + self.assertEqual(('whitespace not allowed',), cm.exception.args) + + wrapper.space_separated = None # type: ignore + self.assertEqual((), wrapper.space_separated) + self.assertFalse('space-separated' in data) + self.assertFalse('Space-Separated' in data) + + wrapper.space_separated = () # type: ignore + self.assertEqual((), wrapper.space_separated) + self.assertFalse('space-separated' in data) + self.assertFalse('Space-Separated' in data) + + def test_dump(self): # type: ignore + # mypy can't cope with the metaprogramming here + data = Deb822ParagraphElement.new_empty_paragraph() + data['Foo'] = 'bar' + data['Baz'] = 'baz' + data['Space-Separated'] = 'baz quux' + data['Required-Field'] = 'required value' + data['Restricted-Field'] = 'restricted value' + + wrapper = self.Wrapper(data) + self.assertEqual(data.dump(), wrapper.dump()) + + wrapper.restricted_field = 'another value' # type: ignore + wrapper.space_separated = ('bar', 'baz', 'quux') # type: ignore + self.assertEqual(data.dump(), wrapper.dump()) + + class LineBasedTest(unittest.TestCase): """Test for _LineBased.{to,from}_str""" @@ -294,7 +420,7 @@ def test_basic_parse_success(self): # type: () -> None - c = copyright.Copyright(sequence=SIMPLE.splitlines()) + c = copyright.Copyright(sequence=SIMPLE.splitlines(True)) self.assertEqual(FORMAT, c.header.format) self.assertEqual(FORMAT, c.header['Format']) self.assertEqual('X Solitaire', c.header.upstream_name) @@ -307,13 +433,13 @@ def test_parse_and_dump(self): # type: () -> None - c = copyright.Copyright(sequence=SIMPLE.splitlines()) + c = copyright.Copyright(sequence=SIMPLE.splitlines(True)) dumped = c.dump() self.assertEqual(SIMPLE, dumped) def test_all_paragraphs(self): # type: () -> None - c = copyright.Copyright(MULTI_LICENSE.splitlines()) + c = copyright.Copyright(MULTI_LICENSE.splitlines(True)) expected = [] # type: List[copyright.AllParagraphTypes] expected.append(c.header) expected.extend(list(c.all_files_paragraphs())) @@ -323,7 +449,7 @@ def test_all_files_paragraphs(self): # type: () -> None - c = copyright.Copyright(sequence=SIMPLE.splitlines()) + c = copyright.Copyright(sequence=SIMPLE.splitlines(True)) self.assertEqual( [('*',), ('debian/*',)], [fp.files for fp in c.all_files_paragraphs()]) @@ -333,7 +459,7 @@ def test_find_files_paragraph(self): # type: () -> None - c = copyright.Copyright(sequence=SIMPLE.splitlines()) + c = copyright.Copyright(sequence=SIMPLE.splitlines(True)) paragraphs = list(c.all_files_paragraphs()) self.assertIs(paragraphs[0], c.find_files_paragraph('Makefile')) @@ -350,17 +476,30 @@ ['bar/*'], 'CompanyB', copyright.License('Apache')) c.add_files_paragraph(files1) c.add_files_paragraph(files2) - self.assertIs(files1, c.find_files_paragraph('foo/bar.cc')) - self.assertIs(files2, c.find_files_paragraph('bar/baz.cc')) + paragraphs = list(c.all_files_paragraphs()) + self.assertIs(paragraphs[0], c.find_files_paragraph('foo/bar.cc')) + self.assertIs(paragraphs[1], c.find_files_paragraph('bar/baz.cc')) self.assertIsNone(c.find_files_paragraph('baz/quux.cc')) self.assertIsNone(c.find_files_paragraph('Makefile')) + self.assertEqual("""\ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ + +Files: foo/* +Copyright: CompanyA +License: ISC + +Files: bar/* +Copyright: CompanyB +License: Apache +""", c.dump()) + def test_all_license_paragraphs(self): # type: () -> None - c = copyright.Copyright(sequence=SIMPLE.splitlines()) + c = copyright.Copyright(sequence=SIMPLE.splitlines(True)) self.assertEqual([], list(c.all_license_paragraphs())) - c = copyright.Copyright(MULTI_LICENSE.splitlines()) + c = copyright.Copyright(MULTI_LICENSE.splitlines(True)) self.assertEqual( [copyright.License('ABC', '[ABC TEXT]'), copyright.License('123', '[123 TEXT]')], @@ -376,7 +515,7 @@ def test_error_on_invalid(self): # type: () -> None - lic = SIMPLE.splitlines() + lic = SIMPLE.splitlines(True) with self.assertRaises(copyright.MachineReadableFormatError) as cm: # missing License field from 1st Files stanza c = copyright.Copyright(sequence=lic[0:10]) @@ -396,7 +535,7 @@ def setUp(self): # type: () -> None - paragraphs = list(deb822.Deb822.iter_paragraphs(SIMPLE.splitlines())) + paragraphs = list(parse_deb822_file(SIMPLE.splitlines(True))) self.formatted = paragraphs[1]['License'] self.parsed = 'GPL-2+\n' + GPL_TWO_PLUS_TEXT self.parsed_lines = self.parsed.splitlines() @@ -506,7 +645,7 @@ def test_typical(self): # type: () -> None - paragraphs = list(deb822.Deb822.iter_paragraphs(SIMPLE.splitlines())) + paragraphs = list(parse_deb822_file(SIMPLE.splitlines(True))) p = paragraphs[1] l = copyright.License.from_str(p['license']) if l is not None: @@ -523,7 +662,7 @@ def test_properties(self): # type: () -> None - d = deb822.Deb822() + d = Deb822ParagraphElement.new_empty_paragraph() d['License'] = 'GPL-2' lp = copyright.LicenseParagraph(d) self.assertEqual('GPL-2', lp['License']) @@ -544,14 +683,14 @@ def test_no_license(self): # type: () -> None - d = deb822.Deb822() + d = Deb822ParagraphElement.new_empty_paragraph() with self.assertRaises(ValueError) as cm: copyright.LicenseParagraph(d) self.assertEqual(('"License" field required',), cm.exception.args) def test_also_has_files(self): # type: () -> None - d = deb822.Deb822() + d = Deb822ParagraphElement.new_empty_paragraph() d['License'] = 'GPL-2\n [LICENSE TEXT]' d['Files'] = '*' with self.assertRaises(ValueError) as cm: @@ -561,11 +700,14 @@ def test_try_set_files(self): # type: () -> None - lp = copyright.LicenseParagraph( - deb822.Deb822({'License': 'GPL-2\n [LICENSE TEXT]'})) + + d = Deb822ParagraphElement.new_empty_paragraph() + d['License'] = 'GPL-2\n [LICENSE TEXT]' + lp = copyright.LicenseParagraph(d) with self.assertRaises(deb822.RestrictedFieldError): lp['Files'] = 'foo/*' + class GlobsToReTest(unittest.TestCase): def setUp(self): @@ -689,7 +831,7 @@ def setUp(self): # type: () -> None - self.prototype = deb822.Deb822() + self.prototype = Deb822ParagraphElement.new_empty_paragraph() self.prototype['Files'] = '*' self.prototype['Copyright'] = 'Foo' self.prototype['License'] = 'ISC' @@ -777,13 +919,13 @@ def test_format_upgrade_no_header(self): # type: () -> None - data = deb822.Deb822() + data = Deb822ParagraphElement.new_empty_paragraph() with self.assertRaises(copyright.NotMachineReadableError): copyright.Header(data=data) def test_format_https_upgrade(self): # type: () -> None - data = deb822.Deb822() + data = Deb822ParagraphElement.new_empty_paragraph() data['Format'] = "http%s" % FORMAT[5:] with self.assertLogs('debian.copyright', level='WARNING') as cm: self.assertIsNotNone(cm) @@ -805,7 +947,7 @@ def test_upstream_contact_single_read(self): # type: () -> None - data = deb822.Deb822() + data = Deb822ParagraphElement.new_empty_paragraph() data['Format'] = FORMAT data['Upstream-Contact'] = 'Foo Bar ' h = copyright.Header(data=data) @@ -813,7 +955,7 @@ def test_upstream_contact_multi1_read(self): # type: () -> None - data = deb822.Deb822() + data = Deb822ParagraphElement.new_empty_paragraph() data['Format'] = FORMAT data['Upstream-Contact'] = 'Foo Bar \n http://bar.com/foo' h = copyright.Header(data=data) @@ -823,7 +965,7 @@ def test_upstream_contact_multi2_read(self): # type: () -> None - data = deb822.Deb822() + data = Deb822ParagraphElement.new_empty_paragraph() data['Format'] = FORMAT data['Upstream-Contact'] = ( '\n Foo Bar \n http://bar.com/foo') @@ -864,5 +1006,13 @@ self.assertFalse('license' in h) # type: ignore +def _no_space(s): + # type: (str) -> str + """Returns s. Raises ValueError if s contains any whitespace.""" + if re.search(r'\s', s): + raise ValueError('whitespace not allowed') + return s + + if __name__ == '__main__': unittest.main() diff -Nru python-debian-0.1.44/lib/debian/tests/test_deb822.py python-debian-0.1.46/lib/debian/tests/test_deb822.py --- python-debian-0.1.44/lib/debian/tests/test_deb822.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/tests/test_deb822.py 2022-07-08 16:45:29.000000000 +0000 @@ -1699,138 +1699,5 @@ self._validate_gpg_info(gpg_info) -def _no_space(s): - # type: (str) -> str - """Returns s. Raises ValueError if s contains any whitespace.""" - if re.search(r'\s', s): - raise ValueError('whitespace not allowed') - return s - - -class RestrictedWrapperTest(unittest.TestCase): - class Wrapper(deb822.RestrictedWrapper): - restricted_field = deb822.RestrictedField('Restricted-Field') - required_field = deb822.RestrictedField('Required-Field', allow_none=False) - space_separated = deb822.RestrictedField( - 'Space-Separated', - from_str=lambda s: tuple((s or '').split()), - to_str=lambda seq: ' '.join(_no_space(s) for s in seq) or None) - - def test_unrestricted_get_and_set(self): - # type: () -> None - data = deb822.Deb822() - data['Foo'] = 'bar' - - wrapper = self.Wrapper(data) - self.assertEqual('bar', wrapper['Foo']) - wrapper['foo'] = 'baz' - self.assertEqual('baz', wrapper['Foo']) - self.assertEqual('baz', wrapper['foo']) - - multiline = 'First line\n Another line' - wrapper['X-Foo-Bar'] = multiline - self.assertEqual(multiline, wrapper['X-Foo-Bar']) - self.assertEqual(multiline, wrapper['x-foo-bar']) - - expected_data = deb822.Deb822() - expected_data['Foo'] = 'baz' - expected_data['X-Foo-Bar'] = multiline - self.assertEqual(expected_data.keys(), data.keys()) - self.assertEqual(expected_data, data) - - def test_trivially_restricted_get_and_set(self): # type: ignore - # mypy can't cope with the metaprogramming here - data = deb822.Deb822() - data['Required-Field'] = 'some value' - - wrapper = self.Wrapper(data) - self.assertEqual('some value', wrapper.required_field) - self.assertEqual('some value', wrapper['Required-Field']) - self.assertEqual('some value', wrapper['required-field']) - self.assertIsNone(wrapper.restricted_field) - - with self.assertRaises(deb822.RestrictedFieldError): - wrapper['Required-Field'] = 'foo' - with self.assertRaises(deb822.RestrictedFieldError): - wrapper['required-field'] = 'foo' - with self.assertRaises(deb822.RestrictedFieldError): - wrapper['Restricted-Field'] = 'foo' - with self.assertRaises(deb822.RestrictedFieldError): - wrapper['Restricted-field'] = 'foo' - - with self.assertRaises(deb822.RestrictedFieldError): - del wrapper['Required-Field'] - with self.assertRaises(deb822.RestrictedFieldError): - del wrapper['required-field'] - with self.assertRaises(deb822.RestrictedFieldError): - del wrapper['Restricted-Field'] - with self.assertRaises(deb822.RestrictedFieldError): - del wrapper['restricted-field'] - - with self.assertRaises(TypeError): - wrapper.required_field = None # type: ignore - - wrapper.restricted_field = 'special value' # type: ignore - self.assertEqual('special value', data['Restricted-Field']) - wrapper.restricted_field = None # type: ignore - self.assertFalse('Restricted-Field' in data) - self.assertIsNone(wrapper.restricted_field) - - wrapper.required_field = 'another value' # type: ignore - self.assertEqual('another value', data['Required-Field']) - - def test_set_already_none_to_none(self): # type: ignore - # mypy can't cope with the metaprogramming here - data = deb822.Deb822() - wrapper = self.Wrapper(data) - wrapper.restricted_field = 'Foo' # type: ignore - wrapper.restricted_field = None # type: ignore - self.assertFalse('restricted-field' in data) - wrapper.restricted_field = None # type: ignore - self.assertFalse('restricted-field' in data) - - def test_processed_get_and_set(self): # type: ignore - # mypy can't cope with the metaprogramming here - data = deb822.Deb822() - data['Space-Separated'] = 'foo bar baz' - - wrapper = self.Wrapper(data) - self.assertEqual(('foo', 'bar', 'baz'), wrapper.space_separated) - wrapper.space_separated = ['bar', 'baz', 'quux'] # type: ignore - self.assertEqual('bar baz quux', data['space-separated']) - self.assertEqual('bar baz quux', wrapper['space-separated']) - self.assertEqual(('bar', 'baz', 'quux'), wrapper.space_separated) - - with self.assertRaises(ValueError) as cm: - wrapper.space_separated = ('foo', 'bar baz') # type: ignore - self.assertEqual(('whitespace not allowed',), cm.exception.args) - - wrapper.space_separated = None # type: ignore - self.assertEqual((), wrapper.space_separated) - self.assertFalse('space-separated' in data) - self.assertFalse('Space-Separated' in data) - - wrapper.space_separated = () # type: ignore - self.assertEqual((), wrapper.space_separated) - self.assertFalse('space-separated' in data) - self.assertFalse('Space-Separated' in data) - - def test_dump(self): # type: ignore - # mypy can't cope with the metaprogramming here - data = deb822.Deb822() - data['Foo'] = 'bar' - data['Baz'] = 'baz' - data['Space-Separated'] = 'baz quux' - data['Required-Field'] = 'required value' - data['Restricted-Field'] = 'restricted value' - - wrapper = self.Wrapper(data) - self.assertEqual(data.dump(), wrapper.dump()) - - wrapper.restricted_field = 'another value' # type: ignore - wrapper.space_separated = ('bar', 'baz', 'quux') # type: ignore - self.assertEqual(data.dump(), wrapper.dump()) - - if __name__ == '__main__': unittest.main() diff -Nru python-debian-0.1.44/lib/debian/tests/test_debfile.py python-debian-0.1.46/lib/debian/tests/test_debfile.py --- python-debian-0.1.44/lib/debian/tests/test_debfile.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/tests/test_debfile.py 2022-07-08 16:45:29.000000000 +0000 @@ -371,50 +371,47 @@ """ test various compression schemes for the data member """ for compression in self.compressions: with self.temp_deb(data=compression) as debname: - deb = debfile.DebFile(debname) - # random test on the data part, just to check that content access - # is OK - all_files = [os.path.normpath(f) for f in deb.data.tgz().getnames()] - for f in self.example_data_files: - testfile = os.path.normpath(str(self.example_data_dir / f)) - self.assertIn(testfile, all_files, + with debfile.DebFile(debname) as deb: + # random test on the data part, just to check that content access + # is OK + all_files = [os.path.normpath(f) for f in deb.data.tgz().getnames()] + for f in self.example_data_files: + testfile = os.path.normpath(str(self.example_data_dir / f)) + self.assertIn(testfile, all_files, + "Data part failed on compression %s" % compression) + self.assertIn(os.path.normpath(str(self.example_data_dir)), all_files, "Data part failed on compression %s" % compression) - self.assertIn(os.path.normpath(str(self.example_data_dir)), all_files, - "Data part failed on compression %s" % compression) - deb.close() def test_control_compression(self): # type: () -> None """ test various compression schemes for the control member """ for compression in self.compressions: with self.temp_deb(data=compression) as debname: - deb = debfile.DebFile(debname) - # random test on the control part - self.assertIn( - 'control', - [os.path.normpath(p) for p in deb.control.tgz().getnames()], - "Control part failed on compression %s" % compression - ) - self.assertIn( - 'md5sums', - [os.path.normpath(p) for p in deb.control.tgz().getnames()], - "Control part failed on compression %s" % compression - ) - deb.close() + with debfile.DebFile(debname) as deb: + # random test on the control part + self.assertIn( + 'control', + [os.path.normpath(p) for p in deb.control.tgz().getnames()], + "Control part failed on compression %s" % compression + ) + self.assertIn( + 'md5sums', + [os.path.normpath(p) for p in deb.control.tgz().getnames()], + "Control part failed on compression %s" % compression + ) def test_data_names(self): # type: () -> None """ test for file list equality """ with self.temp_deb() as debname: - deb = debfile.DebFile(debname) - tgz = deb.data.tgz() - with os.popen("dpkg-deb --fsys-tarfile %s | tar t" % debname) as tar: - dpkg_names = [os.path.normpath(x.strip()) for x in tar.readlines()] - debfile_names = [os.path.normpath(name) for name in tgz.getnames()] - - # skip the root - self.assertEqual(debfile_names[1:], dpkg_names[1:]) - deb.close() + with debfile.DebFile(debname) as deb: + tgz = deb.data.tgz() + with os.popen("dpkg-deb --fsys-tarfile %s | tar t" % debname) as tar: + dpkg_names = [os.path.normpath(x.strip()) for x in tar.readlines()] + debfile_names = [os.path.normpath(name) for name in tgz.getnames()] + + # skip the root + self.assertEqual(debfile_names[1:], dpkg_names[1:]) def _test_file_contents(self, debname, debfilename, origfilename, modes=None, follow_symlinks=False): # type: (str, Union[str, Path], Union[str, Path], Optional[List[str]], bool) -> None @@ -550,30 +547,28 @@ with os.popen("dpkg-deb -f %s" % debname) as dpkg_deb: filecontrol = "".join(dpkg_deb.readlines()) - deb = debfile.DebFile(debname) - self.assertEqual( - not_none(deb.control.get_content("control")).decode("utf-8"), - filecontrol) - self.assertEqual( - deb.control.get_content("control", encoding="utf-8"), - filecontrol) - deb.close() + with debfile.DebFile(debname) as deb: + self.assertEqual( + not_none(deb.control.get_content("control")).decode("utf-8"), + filecontrol) + self.assertEqual( + deb.control.get_content("control", encoding="utf-8"), + filecontrol) def test_md5sums(self): # type: () -> None """test md5 extraction from .debs""" with self.temp_deb() as debname: - deb = debfile.DebFile(debname) - md5b = deb.md5sums() - md5 = deb.md5sums(encoding="UTF-8") - - data = [ - (self.example_data_dir / "test_Changes", "73dbb291e900d8cd08e2bb76012a3829"), - ] - for f, h in data: - self.assertEqual(md5b[str(f).encode('UTF-8')], h) - self.assertEqual(md5[str(f)], h) - deb.close() + with debfile.DebFile(debname) as deb: + md5b = deb.md5sums() + md5 = deb.md5sums(encoding="UTF-8") + + data = [ + (self.example_data_dir / "test_Changes", "73dbb291e900d8cd08e2bb76012a3829"), + ] + for f, h in data: + self.assertEqual(md5b[str(f).encode('UTF-8')], h) + self.assertEqual(md5[str(f)], h) def test_contextmanager(self): # type: () -> None @@ -588,11 +583,10 @@ # type: () -> None """test use of DebFile without the contextmanager""" with self.temp_deb() as debname: - deb = debfile.DebFile(debname) - all_files = deb.data.tgz().getnames() - self.assertTrue(all_files) - self.assertTrue(deb.control.get_content("control")) - deb.close() + with debfile.DebFile(debname) as deb: + all_files = deb.data.tgz().getnames() + self.assertTrue(all_files) + self.assertTrue(deb.control.get_content("control")) if __name__ == '__main__': diff -Nru python-debian-0.1.44/lib/debian/tests/test_repro_deb822.py python-debian-0.1.46/lib/debian/tests/test_repro_deb822.py --- python-debian-0.1.44/lib/debian/tests/test_repro_deb822.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/tests/test_repro_deb822.py 2022-07-08 16:45:29.000000000 +0000 @@ -21,10 +21,9 @@ import collections import contextlib import logging -import sys import textwrap from debian.deb822 import Deb822 -from unittest import TestCase, SkipTest +from unittest import TestCase from debian._deb822_repro import (parse_deb822_file, AmbiguousDeb822FieldKeyError, @@ -61,13 +60,26 @@ # NB: As a side-effect of the implementation, the tests strips '¶' unconditionally. # Please another fancy glyph if you need to test non-standard characters. ROUND_TRIP_CASES = [ - RoundTripParseCase(input='', + RoundTripParseCase(input='\n', is_valid_file=False, error_element_count=0, duplicate_fields=False, paragraph_count=0 ), - RoundTripParseCase(input='A: b', + RoundTripParseCase(input='A: b\n', + is_valid_file=True, + error_element_count=0, + duplicate_fields=False, + paragraph_count=1 + ), + RoundTripParseCase(input=textwrap.dedent('''\ + Source: debhelper + # Trailing-whitespace + # Comment before a field + Build-Depends: po4a + # Ending with an empty field + Empty-Field: + '''), is_valid_file=True, error_element_count=0, duplicate_fields=False, @@ -100,6 +112,8 @@ # will be using the end of line marker through out this paragraph ¶ Package: libdebhelper-perl¶ Priority:optional ¶ + # Allowed for debian/control file + Empty-Field:¶ Section: section ¶ # Field starting with a space + newline (special-case)¶ Depends:¶ @@ -112,7 +126,8 @@ , something ¶ , another¶ # Field that ends without a newline¶ - Architecture: all¶'''), + Architecture: all¶ + '''), paragraph_count=3, is_valid_file=True, duplicate_fields=False, @@ -180,11 +195,6 @@ class FormatPreservingDeb822ParserTests(TestCase): - def setUp(self) -> None: - - if sys.version_info < (3, 9): - raise SkipTest('The format preserving parser assume python 3.9') - def test_round_trip_cases(self): # type: () -> None @@ -244,32 +254,6 @@ " with newlines omitted") logging.info("Successfully passed case " + c) - def test_invalid_input_newlines(self): - # type: () -> None - - # Newlines must be provided consistently - file_input = ["A: B\n", - "B: C", - "C: D\n", - ] - with self.assertRaises(ValueError): - parse_deb822_file(file_input) - - file_input = ["A: B", - "B: C\n", - "C: D\n", - ] - with self.assertRaises(ValueError): - parse_deb822_file(file_input) - - # But it is ok for the last one to be missing a newline (as that is a feature - # of the input that might need to be preserved) - file_input = ["A: B\n", - "B: C\n", - "C: D", - ] - parse_deb822_file(file_input) - def test_deb822_emulation(self): # type: () -> None @@ -366,6 +350,65 @@ self.assertEqual(expected, deb822_file.convert_to_text(), "Mutation should have worked while preserving space + tab") + def test_empty_fields(self): + # type: () -> None + original = textwrap.dedent('''\ + Source: foo + Field: foo + Empty-Field:''') + + deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + + source_paragraph = next(iter(deb822_file)) + self.assertEqual("", source_paragraph['Empty-Field']) + source_paragraph['Another-Empty-Field'] = "" + self.assertEqual("", source_paragraph['Another-Empty-Field']) + list_view = source_paragraph.as_interpreted_dict_view(LIST_SPACE_SEPARATED_INTERPRETATION) + with list_view['Empty-Field'] as empty_field: + self.assertFalse(bool(empty_field)) + + with list_view['Field'] as field: + self.assertTrue(bool(field)) + field.clear() + self.assertFalse(bool(field)) + + expected = textwrap.dedent('''\ + Source: foo + Field: + Empty-Field: + Another-Empty-Field: + ''') + self.assertEqual(expected, deb822_file.convert_to_text(), + "Mutation should have worked and generate a valid file") + + def test_empty_fields_reorder(self): + # type: () -> None + original = textwrap.dedent('''\ + Source: foo + Field: foo + Empty-Field:''') + deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + source_paragraph = next(iter(deb822_file)) + source_paragraph.order_last('Field') + expected = textwrap.dedent('''\ + Source: foo + Empty-Field: + Field: foo + ''') + self.assertEqual(expected, deb822_file.convert_to_text(), + "Mutation should have worked and generate a valid file") + # Re-parse + deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + source_paragraph = next(iter(deb822_file)) + source_paragraph.order_first('Empty-Field') + expected = textwrap.dedent('''\ + Empty-Field: + Source: foo + Field: foo + ''') + self.assertEqual(expected, deb822_file.convert_to_text(), + "Mutation should have worked and generate a valid file") + def test_case_preservation(self): # type: () -> None original = textwrap.dedent('''\ @@ -783,6 +826,94 @@ "Mutation should have worked while preserving " "comments") + def test_remove_paragraph(self): + # type: () -> None + original = textwrap.dedent('''\ + Source: foo + # Comment for RRR + Rules-Requires-Root: no + + Package: bar + ''') + + deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + + binary_paragraph = list(deb822_file)[1] + self.assertEqual('bar', binary_paragraph['Package']) + + deb822_file.remove(binary_paragraph) + + expected = textwrap.dedent('''\ + Source: foo + # Comment for RRR + Rules-Requires-Root: no + ''') + + self.assertEqual(expected, deb822_file.convert_to_text(), + "Mutation should have worked while preserving " + "comments") + + # Verify that we can add another paragraph. + deb822_file.append(Deb822ParagraphElement.from_dict({'Package': 'bloe'})) + + expected = textwrap.dedent('''\ + Source: foo + # Comment for RRR + Rules-Requires-Root: no + + Package: bloe + ''') + + self.assertEqual(expected, deb822_file.convert_to_text(), + "Adding new paragraph should have worked") + + deb822_file.remove(list(deb822_file)[1]) + + source_paragraph = list(deb822_file)[0] + self.assertEqual('foo', source_paragraph['Source']) + + deb822_file.remove(source_paragraph) + + expected = textwrap.dedent('''\ + ''') + + self.assertEqual(expected, deb822_file.convert_to_text(), + "Mutation should have worked while preserving " + "comments") + + original = textwrap.dedent('''\ + Source: foo + # Comment for RRR + Rules-Requires-Root: no + + Package: bar + + # Comment + + Package: la + ''') + + deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + + binary_paragraph = list(deb822_file)[1] + self.assertEqual('bar', binary_paragraph['Package']) + + deb822_file.remove(binary_paragraph) + + expected = textwrap.dedent('''\ + Source: foo + # Comment for RRR + Rules-Requires-Root: no + + # Comment + + Package: la + ''') + + self.assertEqual(expected, deb822_file.convert_to_text(), + "Mutation should have worked while preserving " + "comments") + def test_duplicate_fields(self): # type: () -> None @@ -1646,3 +1777,28 @@ expected_result) as bd_list: bd_list.append_newline() bd_list.append('bar (>= 1.0~)') + + def test_mutate_field_preserves_whitespace(self): + # type: () -> None + + original = textwrap.dedent('''\ + Package: foo + Build-Depends: + debhelper-compat (= 11), + uuid-dev + ''') + deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + source_paragraph = next(iter(deb822_file)) + source_paragraph['Build-Depends'] = source_paragraph['Build-Depends'] + self.assertEqual(original, deb822_file.convert_to_text()) + + original = textwrap.dedent('''\ + Package: foo + Build-Depends: + debhelper-compat (= 11), + uuid-dev + ''') + deb822_file = parse_deb822_file(original.splitlines(keepends=True)) + source_paragraph = next(iter(deb822_file)) + source_paragraph['Build-Depends'] = ' \n debhelper-compat (= 11),\n uuid-dev' + self.assertEqual(original, deb822_file.convert_to_text()) diff -Nru python-debian-0.1.44/lib/debian/tests/test_substvars.py python-debian-0.1.46/lib/debian/tests/test_substvars.py --- python-debian-0.1.44/lib/debian/tests/test_substvars.py 1970-01-01 00:00:00.000000000 +0000 +++ python-debian-0.1.46/lib/debian/tests/test_substvars.py 2022-07-08 16:45:29.000000000 +0000 @@ -0,0 +1,84 @@ +import os +from tempfile import TemporaryDirectory +from unittest import TestCase + +from debian.substvars import Substvars, Substvar + + +class SubstvarsTests(TestCase): + + def test_substvars(self): + # type: () -> None + substvars = Substvars() + + self.assertIsNone(substvars.substvars_path, None) + + # add_dependency automatically creates variables + self.assertTrue('misc:Recommends' not in substvars) + substvars.add_dependency('misc:Recommends', "foo (>= 1.0)") + self.assertEqual(substvars['misc:Recommends'], 'foo (>= 1.0)') + # It can be appended to other variables + substvars['foo'] = 'bar, golf' + substvars.add_dependency('foo', 'dpkg (>= 1.20.0)') + self.assertEqual(substvars['foo'], 'bar, dpkg (>= 1.20.0), golf') + # Exact duplicates are ignored + substvars.add_dependency('foo', 'dpkg (>= 1.20.0)') + self.assertEqual(substvars['foo'], 'bar, dpkg (>= 1.20.0), golf') + + substvar = substvars.as_substvar['foo'] + self.assertEqual(substvar.assignment_operator, "=") + substvar.assignment_operator = "?=" + + with self.assertRaises(ValueError): + # Only "=" and "?=" are allowed + substvar.assignment_operator = 'golf' + + self.assertTrue('foo' in substvars) + del substvars['foo'] + self.assertFalse('foo' in substvars) + + def test_save_raises(self): + # type: () -> None + s = Substvars() + with self.assertRaises(TypeError): + # Should raise because it has no base file + s.save() + + def test_save(self): + # type: () -> None + with TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, "foo.substvars") + # Obviously, this does not exist + self.assertFalse(os.path.exists(filename)) + with Substvars.load_from_path(filename, missing_ok=True) as svars: + svars.add_dependency("misc:Depends", "bar (>= 1.0)") + svars.as_substvar["foo"] = Substvar("anything goes", assignment_operator="?=") + self.assertEqual(svars.substvars_path, filename) + self.assertTrue(os.path.exists(filename)) + + with Substvars.load_from_path(filename) as svars: + # Verify we can actually load the file we just wrote again + self.assertEqual(svars['misc:Depends'], "bar (>= 1.0)") + self.assertEqual(svars.as_substvar["misc:Depends"].assignment_operator, "=") + self.assertEqual(svars['foo'], "anything goes") + self.assertEqual(svars.as_substvar["foo"].assignment_operator, "?=") + + def test_equals(self): + # type: () -> None + foo_a = Substvar("foo", assignment_operator="=") + foo_b = Substvar("foo", assignment_operator="=") + foo_optional_a = Substvar("foo", assignment_operator="?=") + foo_optional_b = Substvar("foo", assignment_operator="?=") + self.assertEqual(foo_a, foo_b) + self.assertEqual(foo_optional_a, foo_optional_b) + + self.assertNotEqual(foo_a, foo_optional_a) + self.assertNotEqual(foo_a, object()) + + substvars_a = Substvars() + substvars_b = Substvars() + substvars_a["foo"] = "bar" + substvars_b["foo"] = "bar" + self.assertEqual(substvars_a, substvars_b) + self.assertNotEqual(substvars_a, object()) + diff -Nru python-debian-0.1.44/lib/debian/_util.py python-debian-0.1.46/lib/debian/_util.py --- python-debian-0.1.44/lib/debian/_util.py 2022-05-29 02:06:57.000000000 +0000 +++ python-debian-0.1.46/lib/debian/_util.py 2022-07-08 16:45:29.000000000 +0000 @@ -23,28 +23,21 @@ class _CaseInsensitiveString(str): """Case insensitive string. """ - __slots__ = ['str_lower', 'str_orig'] + __slots__ = ['str_lower'] if TYPE_CHECKING: # pragma: no cover - # neither pylint nor mypy cope with str_lower/str_orig being defined in __new__ + # neither pylint nor mypy cope with str_lower being defined in __new__ def __init__(self, s): # type: (str) -> None super(_CaseInsensitiveString, self).__init__(s) # type: ignore self.str_lower = '' - self.str_orig = '' def __new__(cls, str_): # type: ignore s = str.__new__(cls, str_) - # The deb822 parser modules need to preserve the original case on key iteration. - # We might as well cache it so it is easy to retrieve with str() - s.str_orig = str_ + # We cache the lower case version of the string to speed up some operations s.str_lower = str_.lower() return s - def __str__(self): - # type: () -> str - return self.str_orig - def __hash__(self): # type: () -> int return hash(self.str_lower) @@ -232,7 +225,12 @@ else: # Primarily as a hint to mypy assert self.tail_node is not None - self.tail_node.insert_after(node) + # Optimize for lots of appends (will happen if you are reading a Packages file) by + # inlining relevant bits of tail_node.insert_after (removing unnecessary checks and + # linking). + assert self.tail_node is not node + node.previous_node = self.tail_node + self.tail_node.next_node = node self.tail_node = node self._size += 1 return node