diff -Nru pip-check-reqs-2.4.2/CHANGELOG.rst pip-check-reqs-2.4.3/CHANGELOG.rst --- pip-check-reqs-2.4.2/CHANGELOG.rst 2022-10-30 13:43:02.000000000 +0000 +++ pip-check-reqs-2.4.3/CHANGELOG.rst 2022-12-24 10:31:52.000000000 +0000 @@ -2,10 +2,15 @@ Release History --------------- -(unreleased) +2.4.3 +- Improves performance on Python 3.11. + +2.4.2 + +- Added support for Python 3.11. - Added `python_requires` to metadata; from now on, releases of - `pip-check-reqs` are marked as compatible with Python 3.6.1 and up. + `pip-check-reqs` are marked as compatible with Python 3.8.0 and up. - Made `--version` flag show interpretter version and path to the package which pip-check-reqs is running from, similar to information shown by `pip --version`. diff -Nru pip-check-reqs-2.4.2/debian/changelog pip-check-reqs-2.4.3/debian/changelog --- pip-check-reqs-2.4.2/debian/changelog 2022-11-03 05:37:38.000000000 +0000 +++ pip-check-reqs-2.4.3/debian/changelog 2023-01-02 21:25:38.000000000 +0000 @@ -1,8 +1,16 @@ +pip-check-reqs (2.4.3-1) unstable; urgency=medium + + * New upstream release + * Bump Standards-Version up to 4.6.2. + * Bump copyright years in debian/copyright and fix upstream contact name. + + -- Francois Marier Mon, 02 Jan 2023 13:25:38 -0800 + pip-check-reqs (2.4.2-1) unstable; urgency=medium * New upstream release * Bump Standards-Version up to 4.6.1. - * Bump copyright years in debian/copyright. + * Bump copyright years in debian/copyright. -- Francois Marier Wed, 02 Nov 2022 22:37:38 -0700 diff -Nru pip-check-reqs-2.4.2/debian/control pip-check-reqs-2.4.3/debian/control --- pip-check-reqs-2.4.2/debian/control 2022-11-03 05:37:38.000000000 +0000 +++ pip-check-reqs-2.4.3/debian/control 2023-01-02 21:25:38.000000000 +0000 @@ -7,7 +7,7 @@ dh-python, python3-all, python3-setuptools, -Standards-Version: 4.6.1 +Standards-Version: 4.6.2 Homepage: https://github.com/r1chardj0n3s/pip-check-reqs Vcs-Browser: https://salsa.debian.org/debian/pip-check-reqs Vcs-Git: https://salsa.debian.org/debian/pip-check-reqs.git diff -Nru pip-check-reqs-2.4.2/debian/copyright pip-check-reqs-2.4.3/debian/copyright --- pip-check-reqs-2.4.2/debian/copyright 2022-11-03 05:37:38.000000000 +0000 +++ pip-check-reqs-2.4.3/debian/copyright 2023-01-02 21:25:38.000000000 +0000 @@ -1,10 +1,10 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: pip-check-reqs -Upstream-Contact: 2020 Adam Dangoor +Upstream-Contact: Adam Dangoor Source: https://pypi.org/project/pip-check-reqs/ Files: * -Copyright: 2020-2022 Adam Dangoor +Copyright: 2020-2023 Adam Dangoor 2015-2020 Richard Jones 2015 Josh Hesketh @@ -12,7 +12,7 @@ License: MIT Files: debian/* -Copyright: 2020-2022 Francois Marier +Copyright: 2020-2023 Francois Marier License: MIT License: MIT diff -Nru pip-check-reqs-2.4.2/pip_check_reqs/common.py pip-check-reqs-2.4.3/pip_check_reqs/common.py --- pip-check-reqs-2.4.2/pip_check_reqs/common.py 2022-10-30 22:46:29.000000000 +0000 +++ pip-check-reqs-2.4.3/pip_check_reqs/common.py 2022-12-24 10:01:48.000000000 +0000 @@ -1,3 +1,5 @@ +"""Common functions.""" + import ast import fnmatch import imp @@ -5,81 +7,98 @@ import os import re import sys - +from dataclasses import dataclass, field from pathlib import Path +from typing import ( + Callable, + Dict, + Generator, + Iterable, + List, + Optional, + Set, + Tuple, + Union, +) -from packaging.utils import canonicalize_name from packaging.markers import Marker +from packaging.utils import NormalizedName, canonicalize_name +from pip._internal.network.session import PipSession +from pip._internal.req.constructors import install_req_from_line +from pip._internal.req.req_file import ParsedRequirement, parse_requirements from . import __version__ -# Between different versions of pip the location of PipSession has changed. -try: - from pip._internal.network.session import PipSession -except ImportError: # pragma: no cover - from pip._internal.download import PipSession -from pip._internal.req.req_file import parse_requirements - - log = logging.getLogger(__name__) +@dataclass class FoundModule: - def __init__(self, modname, filename, locations=None): - self.modname = modname - self.filename = os.path.realpath(filename) - self.locations = locations or [] # filename, lineno - - def __repr__(self): - return 'FoundModule("%s")' % self.modname - - -class ImportVisitor(ast.NodeVisitor): - def __init__(self, options): - super(ImportVisitor, self).__init__() - self.__options = options - self.__modules = {} - self.__location = None + """A module with uses in the source.""" - def set_location(self, location): - self.__location = location - - def visit_Import(self, node): + modname: str + filename: str + locations: List[Tuple[str, int]] = field(default_factory=list) + + def __post_init__(self) -> None: + self.filename = os.path.realpath(self.filename) + + +class _ImportVisitor(ast.NodeVisitor): + def __init__(self, ignore_modules_function: Callable[[str], bool]) -> None: + super().__init__() + self._ignore_modules_function = ignore_modules_function + self._modules: Dict[str, FoundModule] = {} + self._location: Optional[str] = None + + def set_location(self, location: str) -> None: + self._location = location + + # Ignore the name error as we are overriding the method. + def visit_Import( # pylint: disable=invalid-name + self, + node: ast.Import, + ) -> None: for alias in node.names: - self.__addModule(alias.name, node.lineno) + self._add_module(alias.name, node.lineno) - def visit_ImportFrom(self, node): - if node.module == '__future__': + # Ignore the name error as we are overriding the method. + def visit_ImportFrom( # pylint: disable=invalid-name + self, + node: ast.ImportFrom, + ) -> None: + if node.module == "__future__": # not an actual module return for alias in node.names: - if node.module is None: + if node.module is None or node.level != 0: # relative import continue - self.__addModule(node.module + '.' + alias.name, node.lineno) + self._add_module(node.module + "." + alias.name, node.lineno) - def __addModule(self, modname, lineno): - if self.__options.ignore_mods(modname): + def _add_module(self, modname: str, lineno: int) -> None: + if self._ignore_modules_function(modname): return path = None progress = [] modpath = last_modpath = None - for p in modname.split('.'): + for modname_part in modname.split("."): try: - file, modpath, description = imp.find_module(p, path) + _, modpath, _ = imp.find_module(modname_part, path) except ImportError: # the component specified at this point is not importable - # (is just an attr of the module) + # (is just an attribute of the module) # *or* it's not actually installed, so we don't care either break # success! we found *something* - progress.append(p) + progress.append(modname_part) # we might have previously seen a useful path though... - if modpath is None: # pragma: no cover - # the sys module will hit this code path, and os will on 3.11+. - # possibly others will, but I've not discovered them. + if modpath is None: + # the `sys` module will hit this code path, and `os` will on + # 3.11+. + # Possibly others will, but I've not discovered them. modpath = last_modpath break @@ -94,129 +113,137 @@ # the module doesn't actually appear to exist on disk return - modname = '.'.join(progress) - if modname not in self.__modules: - self.__modules[modname] = FoundModule(modname, modpath) - self.__modules[modname].locations.append((self.__location, lineno)) - - def finalise(self): - return self.__modules + modname = ".".join(progress) + if modname not in self._modules: + self._modules[modname] = FoundModule(modname, modpath) + assert isinstance(self._location, str) + self._modules[modname].locations.append((self._location, lineno)) + + def finalise(self) -> Dict[str, FoundModule]: + result = self._modules + return result -def pyfiles(root): - d = os.path.abspath(root) - if not os.path.isdir(d): - n, ext = os.path.splitext(d) - if ext == '.py': - yield d +def pyfiles(root: Path) -> Generator[Path, None, None]: + if root.is_file(): + if root.suffix == ".py": + yield root.absolute() else: - raise ValueError('%s is not a python file or directory' % root) - for root, dirs, files in os.walk(d): - for f in files: - n, ext = os.path.splitext(f) - if ext == '.py': - yield os.path.join(root, f) + raise ValueError(f"{root} is not a python file or directory") + elif root.is_dir(): + for item in root.rglob("*.py"): + yield item.absolute() -def find_imported_modules(options): - vis = ImportVisitor(options) - for path in options.paths: +def find_imported_modules( + paths: Iterable[Path], + ignore_files_function: Callable[[str], bool], + ignore_modules_function: Callable[[str], bool], +) -> Dict[str, FoundModule]: + vis = _ImportVisitor(ignore_modules_function=ignore_modules_function) + for path in paths: for filename in pyfiles(path): - if options.ignore_files(filename): - log.info('ignoring: %s', os.path.relpath(filename)) + if ignore_files_function(str(filename)): + log.info("ignoring: %s", os.path.relpath(filename)) continue - log.debug('scanning: %s', os.path.relpath(filename)) - with open(filename, encoding='utf-8') as f: - content = f.read() - vis.set_location(filename) - vis.visit(ast.parse(content, filename)) + log.debug("scanning: %s", os.path.relpath(filename)) + with open(filename, encoding="utf-8") as file_obj: + content = file_obj.read() + vis.set_location(str(filename)) + vis.visit(ast.parse(content, str(filename))) return vis.finalise() -def find_required_modules(options, requirements_filename: str): +def find_required_modules( + ignore_requirements_function: Callable[ + [Union[str, ParsedRequirement]], bool + ], + skip_incompatible: bool, + requirements_filename: Path, +) -> Set[NormalizedName]: explicit = set() - for requirement in parse_requirements(requirements_filename, - session=PipSession()): - try: - requirement_name = requirement.name - # The type of "requirement" changed between pip versions. - # We exclude the "except" from coverage so that on any pip version we - # can report 100% coverage. - except AttributeError: # pragma: no cover - from pip._internal.req.constructors import install_req_from_line - requirement_name = install_req_from_line( - requirement.requirement, - ).name + for requirement in parse_requirements( + str(requirements_filename), session=PipSession() + ): + requirement_name = install_req_from_line( + requirement.requirement, + ).name + assert isinstance(requirement_name, str) - if options.ignore_reqs(requirement): - log.debug('ignoring requirement: %s', requirement_name) + if ignore_requirements_function(requirement): + log.debug("ignoring requirement: %s", requirement_name) continue - if options.skip_incompatible: + if skip_incompatible: requirement_string = requirement.requirement if not has_compatible_markers(requirement_string): - log.debug('ignoring requirement (incompatible environment ' - 'marker): %s', requirement_string) + log.debug( + "ignoring requirement (incompatible environment " + "marker): %s", + requirement_string, + ) continue - log.debug('found requirement: %s', requirement_name) + log.debug("found requirement: %s", requirement_name) explicit.add(canonicalize_name(requirement_name)) return explicit def has_compatible_markers(full_requirement: str) -> bool: - if ';' not in full_requirement: + if ";" not in full_requirement: return True # No environment marker. - enviroment_marker = full_requirement.split(';')[1] + enviroment_marker = full_requirement.split(";")[1] if not enviroment_marker: return True # Empty environment marker. return Marker(enviroment_marker).evaluate() -def is_package_file(path): - '''Determines whether the path points to a Python package sentinel +def is_package_file(path: str) -> str: + """Determines whether the path points to a Python package sentinel file - the __init__.py or its compiled variants. - ''' - m = re.search(r'(.+)/__init__\.py[co]?$', path) - if m is not None: - return m.group(1) - return '' + """ + search_result = re.search(r"(.+)/__init__\.py[co]?$", path) + if search_result is not None: + return search_result.group(1) + return "" -def ignorer(ignore_cfg): +def ignorer(ignore_cfg: List[str]) -> Callable[..., bool]: if not ignore_cfg: return lambda candidate: False - def f(candidate, ignore_cfg=ignore_cfg): + def ignorer_function( + candidate: Union[str, ParsedRequirement], + ignore_cfg: List[str] = ignore_cfg, + ) -> bool: for ignore in ignore_cfg: - try: - from pip._internal.req.constructors import ( - install_req_from_line, - ) - candidate_path = install_req_from_line( # pragma: no cover + if isinstance(candidate, str): + candidate_path = candidate + else: + optional_candidate_path = install_req_from_line( candidate.requirement, ).name - except (ImportError, AttributeError): - try: - candidate_path = candidate.name - except AttributeError: - candidate_path = candidate + assert isinstance(optional_candidate_path, str) + candidate_path = optional_candidate_path if fnmatch.fnmatch(candidate_path, ignore): return True - elif fnmatch.fnmatch(os.path.relpath(candidate_path), ignore): + if fnmatch.fnmatch(os.path.relpath(candidate_path), ignore): return True return False - return f + return ignorer_function -def version_info(): - return "pip-check-reqs {} from {} (python {})".format( - __version__, - str((Path(__file__) / '..').resolve()), - "{}.{}.{}".format(*sys.version_info), +def version_info() -> str: + major, minor, patch = sys.version_info[:3] + python_version = f"{major}.{minor}.{patch}" + parent_directory = Path(__file__).parent.resolve() + return ( + f"pip-check-reqs {__version__} " + f"from {parent_directory} " + f"(python {python_version})" ) diff -Nru pip-check-reqs-2.4.2/pip_check_reqs/find_extra_reqs.py pip-check-reqs-2.4.3/pip_check_reqs/find_extra_reqs.py --- pip-check-reqs-2.4.2/pip_check_reqs/find_extra_reqs.py 2022-10-30 22:46:29.000000000 +0000 +++ pip-check-reqs-2.4.3/pip_check_reqs/find_extra_reqs.py 2022-12-24 10:31:52.000000000 +0000 @@ -1,55 +1,75 @@ +"""Find extra requirements.""" + +import argparse import collections import importlib.metadata import logging -import pathlib -import optparse import os import sys +from pathlib import Path +from typing import Callable, Iterable, List, Optional, Union +from unittest import mock from packaging.utils import canonicalize_name from pip._internal.commands.show import search_packages_info +from pip._internal.req.req_file import ParsedRequirement + from pip_check_reqs import common from pip_check_reqs.common import version_info log = logging.getLogger(__name__) -def find_extra_reqs(options, requirements_filename): +def find_extra_reqs( + requirements_filename: Path, + paths: Iterable[Path], + ignore_files_function: Callable[[str], bool], + ignore_modules_function: Callable[[str], bool], + ignore_requirements_function: Callable[ + [Union[str, ParsedRequirement]], bool + ], + skip_incompatible: bool, +) -> List[str]: # 1. find files used by imports in the code (as best we can without # executing) - used_modules = common.find_imported_modules(options) + used_modules = common.find_imported_modules( + paths=paths, + ignore_files_function=ignore_files_function, + ignore_modules_function=ignore_modules_function, + ) # 2. find which packages provide which files installed_files = {} - all_pkgs = ( - dist.metadata["Name"] for dist - in importlib.metadata.distributions() - ) - - for package in search_packages_info(all_pkgs): - if isinstance(package, dict): # pragma: no cover - package_name = package['name'] - package_location = package['location'] - package_files = package.get('files', []) or [] - else: # pragma: no cover - package_name = package.name - package_location = package.location - package_files = [] - for item in (package.files or []): - here = pathlib.Path('.').resolve() - item_location_rel = (pathlib.Path(package_location) / item) - item_location = item_location_rel.resolve() - try: - relative_item_location = item_location.relative_to(here) - except ValueError: - # Ideally we would use Pathlib.is_relative_to rather than - # checking for a ValueError, but that is only available in - # Python 3.9+. - relative_item_location = item_location - package_files.append(str(relative_item_location)) + all_pkgs = [ + dist.metadata["Name"] for dist in importlib.metadata.distributions() + ] + + # On Python 3.11 (and maybe higher), setting this environment variable + # dramatically improves speeds. + # See https://github.com/r1chardj0n3s/pip-check-reqs/issues/123. + with mock.patch.dict(os.environ, {"_PIP_USE_IMPORTLIB_METADATA": "False"}): + packages_info = list(search_packages_info(all_pkgs)) + + for package in packages_info: + package_name = package.name + package_location = package.location + package_files = [] + for item in package.files or []: + here = Path(".").resolve() + item_location_rel = Path(package_location) / item + item_location = item_location_rel.resolve() + try: + relative_item_location = item_location.relative_to(here) + except ValueError: + # Ideally we would use Pathlib.is_relative_to rather than + # checking for a ValueError, but that is only available in + # Python 3.9+. + relative_item_location = item_location + package_files.append(str(relative_item_location)) - log.debug('installed package: %s (at %s)', package_name, - package_location) + log.debug( + "installed package: %s (at %s)", package_name, package_location + ) for package_file in package_files: path = os.path.realpath( os.path.join(package_location, package_file), @@ -69,95 +89,117 @@ # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(installed_files[info.filename]) - log.debug('used module: %s (from package %s)', modname, - installed_files[info.filename]) + log.debug( + "used module: %s (from package %s)", + modname, + installed_files[info.filename], + ) used[used_name].append(info) else: log.debug( - 'used module: %s (from file %s, assuming stdlib or local)', - modname, info.filename) + "used module: %s (from file %s, assuming stdlib or local)", + modname, + info.filename, + ) # 4. compare with requirements explicit = common.find_required_modules( - options=options, + ignore_requirements_function=ignore_requirements_function, + skip_incompatible=skip_incompatible, requirements_filename=requirements_filename, ) return [name for name in explicit if name not in used] -def main(): - usage = 'usage: %prog [options] files or directories' - parser = optparse.OptionParser(usage) - parser.add_option("--requirements-file", - dest="requirements_filename", - metavar="PATH", - default="requirements.txt", - help="path to the requirements file " - "(defaults to \"requirements.txt\")") - parser.add_option("-f", - "--ignore-file", - dest="ignore_files", - action="append", - default=[], - help="file paths globs to ignore") - parser.add_option("-m", - "--ignore-module", - dest="ignore_mods", - action="append", - default=[], - help="used module names (globs are ok) to ignore") - parser.add_option("-r", - "--ignore-requirement", - dest="ignore_reqs", - action="append", - default=[], - help="reqs in requirements to ignore") - parser.add_option("-s", - "--skip-incompatible", - dest="skip_incompatible", - action="store_true", - default=False, - help="skip requirements that have incompatible " - "environment markers") - parser.add_option("-v", - "--verbose", - dest="verbose", - action="store_true", - default=False, - help="be more verbose") - parser.add_option("-d", - "--debug", - dest="debug", - action="store_true", - default=False, - help="be *really* verbose") - parser.add_option("-V", "--version", - dest="version", - action="store_true", - default=False, - help="display version information") +def main(arguments: Optional[List[str]] = None) -> None: + """Main entry point.""" + usage = "usage: %prog [options] files or directories" + parser = argparse.ArgumentParser(usage) + parser.add_argument("paths", type=Path, nargs="*") + parser.add_argument( + "--requirements-file", + dest="requirements_filename", + type=Path, + metavar="PATH", + default=Path("requirements.txt"), + help="path to the requirements file " + '(defaults to "requirements.txt")', + ) + parser.add_argument( + "-f", + "--ignore-file", + dest="ignore_files", + action="append", + default=[], + help="file paths globs to ignore", + ) + parser.add_argument( + "-m", + "--ignore-module", + dest="ignore_mods", + action="append", + default=[], + help="used module names (globs are ok) to ignore", + ) + parser.add_argument( + "-r", + "--ignore-requirement", + dest="ignore_reqs", + action="append", + default=[], + help="reqs in requirements to ignore", + ) + parser.add_argument( + "-s", + "--skip-incompatible", + dest="skip_incompatible", + action="store_true", + default=False, + help="skip requirements that have incompatible " "environment markers", + ) + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + default=False, + help="be more verbose", + ) + parser.add_argument( + "-d", + "--debug", + dest="debug", + action="store_true", + default=False, + help="be *really* verbose", + ) + parser.add_argument( + "-V", + "--version", + dest="version", + action="store_true", + default=False, + help="display version information", + ) - (options, args) = parser.parse_args() + parse_result = parser.parse_args(arguments) - if options.version: + if parse_result.version: print(version_info()) sys.exit(0) - if not args: + if not parse_result.paths: parser.error("no source files or directories specified") - sys.exit(2) - options.ignore_files = common.ignorer(options.ignore_files) - options.ignore_mods = common.ignorer(options.ignore_mods) - options.ignore_reqs = common.ignorer(options.ignore_reqs) + ignore_files = common.ignorer(parse_result.ignore_files) + ignore_mods = common.ignorer(parse_result.ignore_mods) + ignore_reqs = common.ignorer(parse_result.ignore_reqs) - options.paths = args - - logging.basicConfig(format='%(message)s') - if options.debug: + logging.basicConfig(format="%(message)s") + if parse_result.debug: level = logging.DEBUG - elif options.verbose: + elif parse_result.verbose: level = logging.INFO else: level = logging.WARN @@ -167,22 +209,19 @@ log.info(version_info()) extras = find_extra_reqs( - options=options, - requirements_filename=options.requirements_filename, + requirements_filename=parse_result.requirements_filename, + paths=parse_result.paths, + ignore_files_function=ignore_files, + ignore_modules_function=ignore_mods, + ignore_requirements_function=ignore_reqs, + skip_incompatible=parse_result.skip_incompatible, ) if extras: - log.warning('Extra requirements:') + log.warning("Extra requirements:") for name in extras: - message = '{name} in {requirements_filename}'.format( - name=name, - requirements_filename=options.requirements_filename, - ) + message = f"{name} in {parse_result.requirements_filename}" log.warning(message) if extras: sys.exit(1) - - -if __name__ == '__main__': # pragma: no cover - main() diff -Nru pip-check-reqs-2.4.2/pip_check_reqs/find_missing_reqs.py pip-check-reqs-2.4.3/pip_check_reqs/find_missing_reqs.py --- pip-check-reqs-2.4.2/pip_check_reqs/find_missing_reqs.py 2022-10-30 22:46:29.000000000 +0000 +++ pip-check-reqs-2.4.3/pip_check_reqs/find_missing_reqs.py 2022-12-24 10:31:52.000000000 +0000 @@ -1,62 +1,73 @@ +"""Find missing requirements.""" + +import argparse import collections import importlib.metadata import logging -import optparse import os -import pathlib import sys +from pathlib import Path +from typing import Callable, Iterable, List, Optional, Tuple +from unittest import mock -from packaging.utils import canonicalize_name +from packaging.utils import NormalizedName, canonicalize_name from pip._internal.commands.show import search_packages_info -# Between different versions of pip the location of PipSession has changed. -try: - from pip._internal.network.session import PipSession -except ImportError: # pragma: no cover - from pip._internal.download import PipSession +from pip._internal.network.session import PipSession +from pip._internal.req.constructors import install_req_from_line from pip._internal.req.req_file import parse_requirements from pip_check_reqs import common -from pip_check_reqs.common import version_info +from pip_check_reqs.common import FoundModule, version_info log = logging.getLogger(__name__) -def find_missing_reqs(options, requirements_filename): +def find_missing_reqs( + requirements_filename: Path, + paths: Iterable[Path], + ignore_files_function: Callable[[str], bool], + ignore_modules_function: Callable[[str], bool], +) -> List[Tuple[NormalizedName, List[FoundModule]]]: # 1. find files used by imports in the code (as best we can without # executing) - used_modules = common.find_imported_modules(options) + used_modules = common.find_imported_modules( + paths=paths, + ignore_files_function=ignore_files_function, + ignore_modules_function=ignore_modules_function, + ) # 2. find which packages provide which files installed_files = {} - all_pkgs = ( - dist.metadata["Name"] for dist - in importlib.metadata.distributions() - ) - - for package in search_packages_info(all_pkgs): - if isinstance(package, dict): # pragma: no cover - package_name = package['name'] - package_location = package['location'] - package_files = package.get('files', []) or [] - else: # pragma: no cover - package_name = package.name - package_location = package.location - package_files = [] - for item in (package.files or []): - here = pathlib.Path('.').resolve() - item_location_rel = (pathlib.Path(package_location) / item) - item_location = item_location_rel.resolve() - try: - relative_item_location = item_location.relative_to(here) - except ValueError: - # Ideally we would use Pathlib.is_relative_to rather than - # checking for a ValueError, but that is only available in - # Python 3.9+. - relative_item_location = item_location - package_files.append(str(relative_item_location)) - - log.debug('installed package: %s (at %s)', package_name, - package_location) + all_pkgs = [ + dist.metadata["Name"] for dist in importlib.metadata.distributions() + ] + + # On Python 3.11 (and maybe higher), setting this environment variable + # dramatically improves speeds. + # See https://github.com/r1chardj0n3s/pip-check-reqs/issues/123. + with mock.patch.dict(os.environ, {"_PIP_USE_IMPORTLIB_METADATA": "False"}): + packages_info = list(search_packages_info(all_pkgs)) + + for package in packages_info: + package_name = package.name + package_location = package.location + package_files = [] + for item in package.files or []: + here = Path(".").resolve() + item_location_rel = Path(package_location) / item + item_location = item_location_rel.resolve() + try: + relative_item_location = item_location.relative_to(here) + except ValueError: + # Ideally we would use Pathlib.is_relative_to rather than + # checking for a ValueError, but that is only available in + # Python 3.9+. + relative_item_location = item_location + package_files.append(str(relative_item_location)) + + log.debug( + "installed package: %s (at %s)", package_name, package_location + ) for package_file in package_files: path = os.path.realpath( os.path.join(package_location, package_file), @@ -75,95 +86,107 @@ # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(installed_files[info.filename]) - log.debug('used module: %s (from package %s)', modname, - installed_files[info.filename]) + log.debug( + "used module: %s (from package %s)", + modname, + installed_files[info.filename], + ) used[used_name].append(info) else: log.debug( - 'used module: %s (from file %s, assuming stdlib or local)', - modname, info.filename) + "used module: %s (from file %s, assuming stdlib or local)", + modname, + info.filename, + ) # 4. compare with requirements explicit = set() for requirement in parse_requirements( - requirements_filename, + str(requirements_filename), session=PipSession(), ): - try: - requirement_name = requirement.name - # The type of "requirement" changed between pip versions. - # We exclude the "except" from coverage so that on any pip version we - # can report 100% coverage. - except AttributeError: # pragma: no cover - from pip._internal.req.constructors import install_req_from_line - requirement_name = install_req_from_line( - requirement.requirement, - ).name + requirement_name = install_req_from_line( + requirement.requirement, + ).name - log.debug('found requirement: %s', requirement_name) + assert isinstance(requirement_name, str) + log.debug("found requirement: %s", requirement_name) explicit.add(canonicalize_name(requirement_name)) - return [(name, used[name]) for name in used if name not in explicit] + result = [(name, used[name]) for name in used if name not in explicit] + return result -def main(): - usage = 'usage: %prog [options] files or directories' - parser = optparse.OptionParser(usage) - parser.add_option("--requirements-file", - dest="requirements_filename", - metavar="PATH", - default="requirements.txt", - help="path to the requirements file " - "(defaults to \"requirements.txt\")") - parser.add_option("-f", - "--ignore-file", - dest="ignore_files", - action="append", - default=[], - help="file paths globs to ignore") - parser.add_option("-m", - "--ignore-module", - dest="ignore_mods", - action="append", - default=[], - help="used module names (globs are ok) to ignore") - parser.add_option("-v", - "--verbose", - dest="verbose", - action="store_true", - default=False, - help="be more verbose") - parser.add_option("-d", - "--debug", - dest="debug", - action="store_true", - default=False, - help="be *really* verbose") - parser.add_option("-V", "--version", - dest="version", - action="store_true", - default=False, - help="display version information") +def main(arguments: Optional[List[str]] = None) -> None: + usage = "usage: %prog [options] files or directories" + parser = argparse.ArgumentParser(usage) + parser.add_argument("paths", type=Path, nargs="*") + parser.add_argument( + "--requirements-file", + dest="requirements_filename", + metavar="PATH", + type=Path, + default="requirements.txt", + help="path to the requirements file " + '(defaults to "requirements.txt")', + ) + parser.add_argument( + "-f", + "--ignore-file", + dest="ignore_files", + action="append", + default=[], + help="file paths globs to ignore", + ) + parser.add_argument( + "-m", + "--ignore-module", + dest="ignore_mods", + action="append", + default=[], + help="used module names (globs are ok) to ignore", + ) + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + default=False, + help="be more verbose", + ) + parser.add_argument( + "-d", + "--debug", + dest="debug", + action="store_true", + default=False, + help="be *really* verbose", + ) + parser.add_argument( + "-V", + "--version", + dest="version", + action="store_true", + default=False, + help="display version information", + ) - (options, args) = parser.parse_args() + parse_result = parser.parse_args(arguments) - if options.version: + if parse_result.version: print(version_info()) sys.exit(0) - if not args: + if not parse_result.paths: parser.error("no source files or directories specified") - sys.exit(2) - options.ignore_files = common.ignorer(options.ignore_files) - options.ignore_mods = common.ignorer(options.ignore_mods) + ignore_files = common.ignorer(parse_result.ignore_files) + ignore_mods = common.ignorer(parse_result.ignore_mods) - options.paths = args - - logging.basicConfig(format='%(message)s') - if options.debug: + logging.basicConfig(format="%(message)s") + if parse_result.debug: level = logging.DEBUG - elif options.verbose: + elif parse_result.verbose: level = logging.INFO else: level = logging.WARN @@ -173,22 +196,24 @@ log.info(version_info()) missing = find_missing_reqs( - options=options, - requirements_filename=options.requirements_filename, + requirements_filename=parse_result.requirements_filename, + paths=parse_result.paths, + ignore_files_function=ignore_files, + ignore_modules_function=ignore_mods, ) if missing: - log.warning('Missing requirements:') + log.warning("Missing requirements:") for name, uses in missing: for use in uses: for filename, lineno in use.locations: - log.warning('%s:%s dist=%s module=%s', - os.path.relpath(filename), lineno, name, - use.modname) + log.warning( + "%s:%s dist=%s module=%s", + os.path.relpath(filename), + lineno, + name, + use.modname, + ) if missing: sys.exit(1) - - -if __name__ == '__main__': # pragma: no cover - main() diff -Nru pip-check-reqs-2.4.2/pip_check_reqs/__init__.py pip-check-reqs-2.4.3/pip_check_reqs/__init__.py --- pip-check-reqs-2.4.2/pip_check_reqs/__init__.py 2022-10-31 22:27:14.000000000 +0000 +++ pip-check-reqs-2.4.3/pip_check_reqs/__init__.py 2022-12-24 10:31:59.000000000 +0000 @@ -1 +1,3 @@ -__version__ = '2.4.2' +"""Package for finding missing and extra requirements.""" + +__version__ = "2.4.3" diff -Nru pip-check-reqs-2.4.2/pip_check_reqs.egg-info/PKG-INFO pip-check-reqs-2.4.3/pip_check_reqs.egg-info/PKG-INFO --- pip-check-reqs-2.4.2/pip_check_reqs.egg-info/PKG-INFO 2022-10-31 22:28:11.000000000 +0000 +++ pip-check-reqs-2.4.3/pip_check_reqs.egg-info/PKG-INFO 2022-12-24 10:32:53.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pip-check-reqs -Version: 2.4.2 +Version: 2.4.3 Summary: Find packages that should or should not be in requirements for a project Home-page: https://github.com/r1chardj0n3s/pip-check-reqs Author: Richard Jones @@ -117,10 +117,15 @@ Release History --------------- -(unreleased) +2.4.3 +- Improves performance on Python 3.11. + +2.4.2 + +- Added support for Python 3.11. - Added `python_requires` to metadata; from now on, releases of - `pip-check-reqs` are marked as compatible with Python 3.6.1 and up. + `pip-check-reqs` are marked as compatible with Python 3.8.0 and up. - Made `--version` flag show interpretter version and path to the package which pip-check-reqs is running from, similar to information shown by `pip --version`. diff -Nru pip-check-reqs-2.4.2/pip_check_reqs.egg-info/requires.txt pip-check-reqs-2.4.3/pip_check_reqs.egg-info/requires.txt --- pip-check-reqs-2.4.2/pip_check_reqs.egg-info/requires.txt 2022-10-31 22:28:11.000000000 +0000 +++ pip-check-reqs-2.4.3/pip_check_reqs.egg-info/requires.txt 2022-12-24 10:32:53.000000000 +0000 @@ -2,6 +2,12 @@ pip>=21.2.4 [dev] -pretend +black +flake8 +isort +mypy +pyenchant +pylint pytest pytest-cov +types-setuptools diff -Nru pip-check-reqs-2.4.2/pip_check_reqs.egg-info/SOURCES.txt pip-check-reqs-2.4.3/pip_check_reqs.egg-info/SOURCES.txt --- pip-check-reqs-2.4.2/pip_check_reqs.egg-info/SOURCES.txt 2022-10-31 22:28:11.000000000 +0000 +++ pip-check-reqs-2.4.3/pip_check_reqs.egg-info/SOURCES.txt 2022-12-24 10:32:53.000000000 +0000 @@ -3,6 +3,7 @@ LICENSE MANIFEST.in README.rst +pyproject.toml requirements.txt setup.cfg setup.py diff -Nru pip-check-reqs-2.4.2/PKG-INFO pip-check-reqs-2.4.3/PKG-INFO --- pip-check-reqs-2.4.2/PKG-INFO 2022-10-31 22:28:11.721503000 +0000 +++ pip-check-reqs-2.4.3/PKG-INFO 2022-12-24 10:32:53.536632300 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pip_check_reqs -Version: 2.4.2 +Version: 2.4.3 Summary: Find packages that should or should not be in requirements for a project Home-page: https://github.com/r1chardj0n3s/pip-check-reqs Author: Richard Jones @@ -117,10 +117,15 @@ Release History --------------- -(unreleased) +2.4.3 +- Improves performance on Python 3.11. + +2.4.2 + +- Added support for Python 3.11. - Added `python_requires` to metadata; from now on, releases of - `pip-check-reqs` are marked as compatible with Python 3.6.1 and up. + `pip-check-reqs` are marked as compatible with Python 3.8.0 and up. - Made `--version` flag show interpretter version and path to the package which pip-check-reqs is running from, similar to information shown by `pip --version`. diff -Nru pip-check-reqs-2.4.2/pyproject.toml pip-check-reqs-2.4.3/pyproject.toml --- pip-check-reqs-2.4.2/pyproject.toml 1970-01-01 00:00:00.000000000 +0000 +++ pip-check-reqs-2.4.3/pyproject.toml 2022-11-22 10:13:14.000000000 +0000 @@ -0,0 +1,100 @@ +[tool.pylint] + + [tool.pylint.'MASTER'] + + # Pickle collected data for later comparisons. + persistent = true + + # Use multiple processes to speed up Pylint. + jobs = 0 + + # List of plugins (as comma separated values of python modules names) to load, + # usually to register additional checkers. + load-plugins = [ + 'pylint.extensions.docparams', + 'pylint.extensions.no_self_use', + ] + + # Allow loading of arbitrary C extensions. Extensions are imported into the + # active Python interpreter and may run arbitrary code. + unsafe-load-any-extension = false + + [tool.pylint.'MESSAGES CONTROL'] + + # Enable the message, report, category or checker with the given id(s). You can + # either give multiple identifier separated by comma (,) or put this option + # multiple time (only on the command line, not in the configuration file where + # it should appear only once). See also the "--disable" option for examples. + enable = [ + 'spelling', + 'useless-suppression', + ] + + # Disable the message, report, category or checker with the given id(s). You + # can either give multiple identifiers separated by comma (,) or put this + # option multiple times (only on the command line, not in the configuration + # file where it should appear only once).You can also use "--disable=all" to + # disable everything first and then reenable specific checks. For example, if + # you want to run only the similarities checker, you can use "--disable=all + # --enable=similarities". If you want to run only the classes checker, but have + # no Warning level messages displayed, use"--disable=all --enable=classes + # --disable=W" + + disable = [ + 'too-few-public-methods', + 'too-many-locals', + 'too-many-arguments', + 'too-many-instance-attributes', + 'too-many-return-statements', + 'too-many-lines', + 'locally-disabled', + # Let flake8 handle long lines + 'line-too-long', + # Let flake8 handle unused imports + 'unused-import', + # Let isort deal with sorting + 'ungrouped-imports', + # We don't need everything to be documented because of mypy + 'missing-type-doc', + 'missing-return-type-doc', + # Too difficult to please + 'duplicate-code', + # Let isort handle imports + 'wrong-import-order', + # It would be nice to add this, but it's too much work + "missing-function-docstring", + # We will remove this in issue 97 + "deprecated-module", + ] + + [tool.pylint.'FORMAT'] + + # Allow the body of an if to be on the same line as the test if there is no + # else. + single-line-if-stmt = false + + [tool.pylint.'SPELLING'] + + # Spelling dictionary name. Available dictionaries: none. To make it working + # install python-enchant package. + spelling-dict = 'en_US' + + # A path to a file that contains private dictionary; one word per line. + spelling-private-dict-file = 'spelling_private_dict.txt' + + # Tells whether to store unknown words to indicated private dictionary in + # --spelling-private-dict-file option instead of raising a message. + spelling-store-unknown-words = 'no' + +[tool.black] + +line-length = 79 + +[tool.mypy] + +strict = true + +[tool.isort] + +multi_line_output = 3 +include_trailing_comma = true diff -Nru pip-check-reqs-2.4.2/setup.py pip-check-reqs-2.4.3/setup.py --- pip-check-reqs-2.4.2/setup.py 2022-10-31 22:27:52.000000000 +0000 +++ pip-check-reqs-2.4.3/setup.py 2022-11-21 02:50:25.000000000 +0000 @@ -1,9 +1,10 @@ -from setuptools import setup from codecs import open from os import path from pathlib import Path from typing import List +from setuptools import setup + from pip_check_reqs import __version__ here = path.abspath(path.dirname(__file__)) @@ -14,55 +15,56 @@ Return requirements from a requirements file. This expects a requirements file with no ``--find-links`` lines. """ - lines = requirements_file.read_text().strip().split('\n') - return [line for line in lines if not line.startswith('#')] + lines = requirements_file.read_text().strip().split("\n") + return [line for line in lines if not line.startswith("#")] -with open(path.join(here, 'README.rst'), encoding='utf-8') as f: +with open(path.join(here, "README.rst"), encoding="utf-8") as f: long_description = f.read() -with open(path.join(here, 'CHANGELOG.rst'), encoding='utf-8') as f: +with open(path.join(here, "CHANGELOG.rst"), encoding="utf-8") as f: long_description += f.read() INSTALL_REQUIRES = _get_dependencies( - requirements_file=Path('requirements.txt'), + requirements_file=Path("requirements.txt"), ) DEV_REQUIRES = _get_dependencies( - requirements_file=Path('test-requirements.txt'), + requirements_file=Path("test-requirements.txt"), ) setup( - name='pip_check_reqs', + name="pip_check_reqs", version=__version__, description=( - 'Find packages that should or should not be in requirements for a ' - 'project'), + "Find packages that should or should not be in requirements for a " + "project" + ), long_description=long_description, - url='https://github.com/r1chardj0n3s/pip-check-reqs', - author='Richard Jones', - author_email='r1chardj0n3s@gmail.com', - maintainer='Adam Dangoor', - maintainer_email='adamdangoor@gmail.com', - license='MIT', + url="https://github.com/r1chardj0n3s/pip-check-reqs", + author="Richard Jones", + author_email="r1chardj0n3s@gmail.com", + maintainer="Adam Dangoor", + maintainer_email="adamdangoor@gmail.com", + license="MIT", classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Build Tools', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], python_requires=">=3.8.0", - packages=['pip_check_reqs'], + packages=["pip_check_reqs"], entry_points={ - 'console_scripts': [ - 'pip-missing-reqs=pip_check_reqs.find_missing_reqs:main', - 'pip-extra-reqs=pip_check_reqs.find_extra_reqs:main', + "console_scripts": [ + "pip-missing-reqs=pip_check_reqs.find_missing_reqs:main", + "pip-extra-reqs=pip_check_reqs.find_extra_reqs:main", ], }, install_requires=INSTALL_REQUIRES, - extras_require={'dev': DEV_REQUIRES}, + extras_require={"dev": DEV_REQUIRES}, ) diff -Nru pip-check-reqs-2.4.2/test-requirements.txt pip-check-reqs-2.4.3/test-requirements.txt --- pip-check-reqs-2.4.2/test-requirements.txt 2022-10-30 13:43:02.000000000 +0000 +++ pip-check-reqs-2.4.3/test-requirements.txt 2022-11-22 10:13:14.000000000 +0000 @@ -1,3 +1,9 @@ -pretend +black +flake8 +isort +mypy +pyenchant +pylint pytest pytest-cov +types-setuptools