diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/backportpackage ubuntu-dev-tools-0.187~bpo20.04.1/backportpackage --- ubuntu-dev-tools-0.185~bpo20.04.1/backportpackage 2021-11-03 16:32:20.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/backportpackage 2021-12-05 15:20:37.000000000 +0000 @@ -29,6 +29,7 @@ import lsb_release from httplib2 import Http, HttpLib2Error +from distro_info import DebianDistroInfo, UbuntuDistroInfo from ubuntutools.archive import (DebianSourcePackage, UbuntuSourcePackage, DownloadError) @@ -80,7 +81,7 @@ metavar='MESSAGE', default="No-change", help='Changelog message to use instead of "No-change" ' - '(default: No-change backport to DEST)') + '(default: No-change backport to DEST.)') parser.add_option('-b', '--build', default=False, action='store_true', @@ -225,10 +226,25 @@ distribution = codename_to_distribution(release) if not distribution: error('Unknown release codename %s' % release) - series = Distribution(distribution.lower()).\ - getSeries(name_or_version=release) + if distribution == 'Debian': + debian_distro_info = DebianDistroInfo() + debian_codenames = debian_distro_info.supported() + if release in debian_codenames: + release_version = debian_distro_info.version(release) + if not release_version: + error(f"Can't find the release version for {release}") + backport_version = "{}~bpo{}+1".format( + version, release_version + ) + else: + error(f"{release} is not a supported release ({debian_codenames})") + elif distribution == 'Ubuntu': + series = Distribution(distribution.lower()).\ + getSeries(name_or_version=release) - backport_version = version + ('~%s%s.1' % (distribution.lower(), series.version)) + backport_version = version + ('~bpo%s.1' % (series.version)) + else: + error('Unknown distribution «%s» for release «%s»' % (distribution, release)) if suffix is not None: backport_version += suffix elif upload and upload.startswith('ppa:'): @@ -327,7 +343,7 @@ old_version = get_old_version(pkg.source, release) bp_dist = get_backport_dist(release, release_pocket) - changelog = '%s backport to %s' % (message, release,) + changelog = '%s backport to %s.' % (message, release,) if close: changelog += ' (LP: #%s)' % (close,) check_call(['dch', @@ -382,9 +398,15 @@ if not opts.dest_releases: distinfo = lsb_release.get_distro_information() try: - opts.dest_releases = [distinfo['CODENAME']] + current_distro = distinfo['ID'] except KeyError: error('No destination release specified and unable to guess yours.') + if current_distro == "Ubuntu": + opts.dest_releases = [UbuntuDistroInfo().lts()] + if current_distro == "Debian": + opts.dest_releases = [DebianDistroInfo().stable()] + else: + error(f"Unknown distribution {current_distro}, can't guess target release") if opts.workdir: workdir = os.path.expanduser(opts.workdir) diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/debian/changelog ubuntu-dev-tools-0.187~bpo20.04.1/debian/changelog --- ubuntu-dev-tools-0.185~bpo20.04.1/debian/changelog 2021-11-03 16:32:27.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/debian/changelog 2021-12-09 15:27:00.000000000 +0000 @@ -1,9 +1,47 @@ -ubuntu-dev-tools (0.185~bpo20.04.1) focal-backports; urgency=medium +ubuntu-dev-tools (0.187~bpo20.04.1) focal-backports; urgency=medium - * Rebuild for focal-backports. LP: #1947192 - + backport changes: revert to debhelper compat 12. + * Rebuild for focal-backports. - -- Mattia Rizzolo Wed, 03 Nov 2021 17:32:27 +0100 + -- Mattia Rizzolo Thu, 09 Dec 2021 16:27:00 +0100 + +ubuntu-dev-tools (0.187) unstable; urgency=medium + + [ Paride Legovini ] + * mk-sbuild: + + Add support for zfs-snapshot schroots. LP: #1945349 + + [ Mattia Rizzolo ] + * mk-sbuild: + + Apply patch from Peter Pentchev to avoid a broken log message. + Closes: #968316 + * backportpackage: + + Support backporting to Debian releases. Closes: #776442; LP: #974132 + + Fix the guessing algorithm for the target release: + - for Debian: pick the current stable release. + - for Ubuntu: pick the current LTS release. + + [ Unit 193 ] + * backportpackage: + + Change the generated Ubuntu version following the new policy from the + Backporters team. + + [ Dan Streetman ] + * misc: + + Refactor download progress bar code. + + Save files that have Content-Encoding correctly, + such as the changes file from upload queue packages. + * pullpkg: + + Extract source packages pulled from upload queue. + * hugdaylist: + + Remove long unused and non-working script. + + -- Mattia Rizzolo Sun, 05 Dec 2021 15:58:15 +0100 + +ubuntu-dev-tools (0.186) unstable; urgency=medium + + * Replace nose with pytest (see: #997758). + + -- Stefano Rivera Sun, 24 Oct 2021 16:10:44 -0700 ubuntu-dev-tools (0.185) unstable; urgency=medium diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/debian/control ubuntu-dev-tools-0.187~bpo20.04.1/debian/control --- ubuntu-dev-tools-0.185~bpo20.04.1/debian/control 2021-11-03 16:32:27.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/debian/control 2021-12-05 15:20:37.000000000 +0000 @@ -8,7 +8,7 @@ Mattia Rizzolo , Build-Depends: dctrl-tools, - debhelper-compat (= 12), + debhelper-compat (= 13), devscripts (>= 2.11.0~), dh-make, dh-python, @@ -22,7 +22,7 @@ python3-distro-info, python3-httplib2, python3-launchpadlib, - python3-nose, + python3-pytest, python3-requests , python3-setuptools, Standards-Version: 4.6.0 @@ -90,7 +90,6 @@ - dch-repeat - used to repeat a change log into an older release. - grab-merge - grabs a merge from merges.ubuntu.com easily. - grep-merges - search for pending merges from Debian. - - hugdaylist - compile HugDay lists from bug list URLs. - import-bug-from-debian - copy a bug from the Debian BTS to Launchpad - merge-changelog - manually merges two Debian changelogs with the same base version. diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/debian/copyright ubuntu-dev-tools-0.187~bpo20.04.1/debian/copyright --- ubuntu-dev-tools-0.185~bpo20.04.1/debian/copyright 2021-07-17 15:27:49.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/debian/copyright 2021-12-05 15:20:37.000000000 +0000 @@ -72,13 +72,11 @@ Files: doc/bitesize.1 doc/grab-merge.1 - doc/hugdaylist.1 doc/merge-changelog.1 doc/setup-packaging-environment.1 doc/syncpackage.1 bitesize grab-merge - hugdaylist merge-changelog setup-packaging-environment syncpackage @@ -109,7 +107,6 @@ grep-merges mk-sbuild ubuntu-build - ubuntutools/lp/libsupport.py ubuntutools/lp/lpapicache.py ubuntutools/misc.py Copyright: 2007-2010, Canonical Ltd. diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/debian/rules ubuntu-dev-tools-0.187~bpo20.04.1/debian/rules --- ubuntu-dev-tools-0.185~bpo20.04.1/debian/rules 2021-11-03 16:32:20.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/debian/rules 2021-12-05 15:20:37.000000000 +0000 @@ -8,7 +8,7 @@ override_dh_auto_test: ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) flake8 -v --max-line-length=99 - nosetests3 -v ubuntutools + python3 -m pytest -v ubuntutools endif %: diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/debian/tests/control ubuntu-dev-tools-0.187~bpo20.04.1/debian/tests/control --- ubuntu-dev-tools-0.185~bpo20.04.1/debian/tests/control 2021-11-03 16:32:20.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/debian/tests/control 2021-12-05 15:20:37.000000000 +0000 @@ -3,10 +3,10 @@ flake8, Restrictions: allow-stderr -Test-Command: nosetests3 -v ubuntutools +Test-Command: python3 -m pytest -v ubuntutools Depends: dh-make, - python3-nose, + python3-pytest, python3-setuptools, @, Restrictions: allow-stderr diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/doc/hugdaylist.1 ubuntu-dev-tools-0.187~bpo20.04.1/doc/hugdaylist.1 --- ubuntu-dev-tools-0.185~bpo20.04.1/doc/hugdaylist.1 2017-04-30 20:39:45.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/doc/hugdaylist.1 1970-01-01 00:00:00.000000000 +0000 @@ -1,26 +0,0 @@ -.TH HUGDAYLIST "1" "August 27, 2008" "ubuntu-dev-tools" - -.SH NAME -hugdaylist \- produce MoinMoin wiki formatted tables based on a Launchpad bug list - -.SH SYNOPSIS -.B hugdaylist [\fB\-n\fP|\fB\-\-number \fP] \fBlaunchpad-buglist-url\fP - -.SH DESCRIPTION -\fBhugdaylist\fP produces MoinMoin wiki formatted tables based on a -Launchpad bug list - -.SH OPTIONS -.TP -\fB\-\-number=\fP -This option allows you to specify the number of entries to output. -.TP -\fBlaunchpad-buglist-url\fP -Required, this option is a URL pointing to a launchpad bug list. - -.SH AUTHOR -\fBhugdaylist\fP has been written by Canonical Ltd., Daniel Holbach - and Jonathan Patrick Davies . -This manual page was written by Ryan Kavanagh . -.PP -Both are released under the GNU General Public License, version 3. diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/hugdaylist ubuntu-dev-tools-0.187~bpo20.04.1/hugdaylist --- ubuntu-dev-tools-0.185~bpo20.04.1/hugdaylist 2021-02-03 13:24:39.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/hugdaylist 1970-01-01 00:00:00.000000000 +0000 @@ -1,141 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- -# -# Copyright (C) 2007 Canonical Ltd., Daniel Holbach -# Copyright (C) 2008 Jonathan Patrick Davies -# -# ################################################################## -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; version 3. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# See file /usr/share/common-licenses/GPL-3 for more details. -# -# ################################################################## -# -# -# hugdaylist - produces MoinMoin wiki formatted tables based on a Launchpad bug -# list. -# -# hugdaylist -# - produces lists like https://wiki.ubuntu.com/UbuntuBugDay/20070912?action=raw -# -# hugdaylist -n -# - will only list URLs. - -import sys -from optparse import OptionParser - -from launchpadlib.launchpad import Launchpad - -from ubuntutools.lp.libsupport import translate_web_api - -from ubuntutools import getLogger -Logger = getLogger() - - -def check_args(): - howmany = -1 - url = "" - - # Our usage options. - usage = "usage: %prog [-n ] launchpad-buglist-url" - opt_parser = OptionParser(usage) - - # Options - namely just the number of bugs to output. - opt_parser.add_option("-n", "--number", type="int", - dest="number", help="Number of entries to output.") - - # Parse arguments. - (options, args) = opt_parser.parse_args() - - # Check if we want a number other than the default. - howmany = options.number - - # Check that we have an URL. - if not args: - Logger.error("An URL pointing to a Launchpad bug list is required.") - opt_parser.print_help() - sys.exit(1) - else: - url = args[0] - - return (howmany, url) - - -def filter_unsolved(task): - # TODO: don't use this filter here, only check status and assignee of - # the given task - # Filter out special types of bugs: - # - https://wiki.ubuntu.com/Bugs/HowToTriage#Special%20types%20of%20bugs - # this is expensive, parse name out of self_link instead? - subscriptions = set(s.person.name for s in task.bug.subscriptions) - if (task.status != "Fix Committed" and - (not task.assignee or task.assignee.name in ['motu', 'desktop-bugs']) and - 'ubuntu-sponsors' not in subscriptions and - 'ubuntu-archive' not in subscriptions): - return True - return False - - -def main(): - (howmany, url) = check_args() - if len(url.split("?", 1)) == 2: - # search options not supported, because there is no mapping web ui - # options <-> API options - Logger.error("Options in url are not supported, url: %s" % url) - sys.exit(1) - - launchpad = None - try: - launchpad = Launchpad.login_with("ubuntu-dev-tools", 'production') - except IOError as error: - Logger.exception(error) - sys.exit(1) - - api_url = translate_web_api(url, launchpad) - try: - product = launchpad.load(api_url) - except Exception as error: - response = getattr(error, "response", {}) - if response.get("status", None) == "404": - Logger.error("The URL at '%s' does not appear to be a " - "valid url to a product" % url) - sys.exit(1) - else: - raise - - bug_list = [b for b in product.searchTasks() if filter_unsolved(b)] - - if not bug_list: - Logger.info("Bug list of %s is empty." % url) - sys.exit(0) - if howmany == -1: - howmany = len(bug_list) - - Logger.info(""" -## || This task is done || somebody || || -## || This task is assigned || somebody || || -## || This task isn't || ... || || -## || This task is blocked on something || somebody || || - -|| Bug || Subject || Triager ||""") - - for i in list(bug_list)[:howmany]: - bug = i.bug - Logger.info('|| [%s %s] || %s || ||' % - (bug.web_link, bug.id, bug.title)) - - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - Logger.error("Aborted.") - sys.exit(1) diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/mk-sbuild ubuntu-dev-tools-0.187~bpo20.04.1/mk-sbuild --- ubuntu-dev-tools-0.185~bpo20.04.1/mk-sbuild 2021-11-03 16:32:20.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/mk-sbuild 2021-12-05 15:20:37.000000000 +0000 @@ -26,7 +26,7 @@ # ################################################################## # # This script creates chroots designed to be used in a snapshot mode -# (with LVM, btrfs, overlay, overlay or aufs) with schroot and sbuild. +# (with LVM, btrfs, zfs, overlay, overlay or aufs) with schroot and sbuild. # Much love to "man sbuild-setup", https://wiki.ubuntu.com/PbuilderHowto, # and https://help.ubuntu.com/community/SbuildLVMHowto. # @@ -51,6 +51,7 @@ echo " --name=NAME Base name for the schroot (arch is appended)" echo " --personality=PERSONALITY What personality to use (defaults to match --arch)" echo " --vg=VG use LVM snapshots, with group VG" + echo " --zfs-dataset=DATASET use ZFS snapshots, with parent dataset DATASET" echo " --debug Turn on script debugging" echo " --skip-updates Do not include -updates pocket in sources.list" echo " --skip-security Do not include -security pocket in sources.list" @@ -76,8 +77,9 @@ echo " (defaults to determining from release name)" echo " --target=ARCH Target architecture for cross-building" echo " --type=SCHROOT_TYPE Define the schroot type:" - echo " 'directory'(default), 'file', or 'btrfs-snapshot'" + echo " 'directory' (default), 'file', or 'btrfs-snapshot'." echo " 'lvm-snapshot' is selected via --vg" + echo " 'zfs-snapshot' is selected via --zfs-dataset" echo "" echo "Configuration (via ~/.mk-sbuild.rc)" echo " LV_SIZE Size of source LVs (default ${LV_SIZE})" @@ -136,6 +138,7 @@ personality: distro: vg: + zfs-dataset: type: target: ccache-dir: @@ -244,6 +247,10 @@ VG="$2" shift 2 ;; + --zfs-dataset) + ZFS_PARENT_DATASET="$2" + shift 2 + ;; --type) SCHROOT_TYPE="$2" shift 2 @@ -505,6 +512,9 @@ # To build the LV, we need to know which volume group to use if [ -n "$VG" ]; then SCHROOT_TYPE="lvm-snapshot" + # To build the ZFS dataset, we need to know which parent to use + elif [ -n "$ZFS_PARENT_DATASET" ]; then + SCHROOT_TYPE="zfs-snapshot" else SCHROOT_TYPE="directory" fi @@ -551,7 +561,7 @@ # Set up some variables for use in the paths and names CHROOT_PATH="${SOURCE_CHROOTS_TGZ}/${CHROOT_NAME}.tgz" ;; -"btrfs-snapshot") +"btrfs-snapshot" | "zfs-snapshot") if [ ! -d "${SOURCE_CHROOTS_DIR}" ]; then sudo mkdir -p "${SOURCE_CHROOTS_DIR}" fi @@ -757,6 +767,19 @@ fi sudo btrfs subvolume create "${MNT}" ;; + +"zfs-snapshot") + ZFS_DATASET="${ZFS_PARENT_DATASET}/${CHROOT_NAME}" + if sudo zfs list "${ZFS_DATASET}" >/dev/null 2>&1; then + echo "E: ZFS dataset ${ZFS_DATASET} already exists; aborting" >&2 + exit 1 + fi + sudo zfs create -p -o mountpoint=legacy "${ZFS_DATASET}" + + # Mount + MNT=`mktemp -d -t schroot-XXXXXX` + sudo mount -t zfs "${ZFS_DATASET}" "${MNT}" + ;; "file") MNT=`mktemp -d -t schroot-XXXXXX` esac @@ -909,6 +932,12 @@ btrfs-snapshot-directory=CHROOT_SNAPSHOT_DIR EOM ;; +zfs-snapshot) + cat >> "${TEMP_SCHROOTCONF}" < /dev/null rm -f "$TEMP_SCHROOTCONF" @@ -999,7 +1029,7 @@ sudo chmod a+x "$MNT"/finish.sh case "$SCHROOT_TYPE" in -"lvm-snapshot") +"lvm-snapshot"|"zfs-snapshot") sudo umount "$MNT" rmdir "$MNT" ;; @@ -1023,7 +1053,7 @@ echo " To CHANGE the golden image: sudo schroot -c source:${CHROOT_NAME} -u root" echo " To ENTER an image snapshot: schroot -c ${CHROOT_NAME}" echo " To BUILD within a snapshot: sbuild -A -d ${CHROOT_NAME} PACKAGE*.dsc" -if [ "$CHROOT_ARCH" != "$TARGET_ARCH" ] ; then +if [ -n "$TARGET_ARCH" ] && [ "$CHROOT_ARCH" != "$TARGET_ARCH" ] ; then echo " To BUILD for ${TARGET_ARCH}: sbuild -A -d ${CHROOT_NAME} --host ${TARGET_ARCH} PACKAGE*.dsc" fi echo "" diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/setup.py ubuntu-dev-tools-0.187~bpo20.04.1/setup.py --- ubuntu-dev-tools-0.185~bpo20.04.1/setup.py 2021-02-03 13:24:39.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/setup.py 2021-12-05 15:20:37.000000000 +0000 @@ -21,7 +21,6 @@ 'dch-repeat', 'grab-merge', 'grep-merges', - 'hugdaylist', 'import-bug-from-debian', 'merge-changelog', 'mk-sbuild', diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/test-requirements.txt ubuntu-dev-tools-0.187~bpo20.04.1/test-requirements.txt --- ubuntu-dev-tools-0.185~bpo20.04.1/test-requirements.txt 2021-11-03 16:32:20.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/test-requirements.txt 2021-12-05 15:20:37.000000000 +0000 @@ -1,3 +1,4 @@ coverage flake8 >= 3.8.0 -nose +pytest +pytest-cov diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/tox.ini ubuntu-dev-tools-0.187~bpo20.04.1/tox.ini --- ubuntu-dev-tools-0.185~bpo20.04.1/tox.ini 2021-11-03 16:32:20.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/tox.ini 2021-12-05 15:20:37.000000000 +0000 @@ -1,5 +1,5 @@ [tox] -envlist = flake8,nose +envlist = flake8,pytest skipsdist = True [testenv] @@ -14,8 +14,8 @@ [testenv:flake8] commands = flake8 {posargs} -[testenv:nose] -commands = nosetests -v --with-coverage --cover-package=ubuntutools {posargs:ubuntutools} +[testenv:pytest] +commands = pytest -v --cov=ubuntutools {posargs:ubuntutools} [flake8] verbose = 2 diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/archive.py ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/archive.py --- ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/archive.py 2021-09-18 15:20:06.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/archive.py 2021-12-09 15:27:00.000000000 +0000 @@ -492,7 +492,7 @@ stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if result.returncode != 0: Logger.error('Source unpack failed.') - sys.exit(1) + Logger.debug(result.stdout) def debdiff(self, newpkg, diffstat=False): """Write a debdiff comparing this src pkg to a newer one. diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/lp/libsupport.py ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/lp/libsupport.py --- ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/lp/libsupport.py 2019-09-10 12:14:20.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/lp/libsupport.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -# -# libsupport.py - functions which add launchpadlib support to the Ubuntu -# Developer Tools package. -# -# Copyright (C) 2009 Markus Korn -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 3 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# Please see the /usr/share/common-licenses/GPL file for the full text of -# the GNU General Public License license. -# - -from urllib.parse import urlsplit, urlencode, urlunsplit - - -def query_to_dict(query_string): - result = dict() - options = filter(None, query_string.split("&")) - for opt in options: - key, value = opt.split("=") - result.setdefault(key, set()).add(value) - return result - - -def translate_web_api(url, launchpad): - scheme, netloc, path, query, fragment = urlsplit(url) - query = query_to_dict(query) - - differences = set(netloc.split('.')).symmetric_difference( - set(launchpad._root_uri.host.split('.'))) - if ('staging' in differences or 'edge' in differences): - raise ValueError("url conflict (url: %s, root: %s" % - (url, launchpad._root_uri)) - if path.endswith("/+bugs"): - path = path[:-6] - if "ws.op" in query: - raise ValueError("Invalid web url, url: %s" % url) - query["ws.op"] = "searchTasks" - scheme, netloc, api_path, _, _ = urlsplit(str(launchpad._root_uri)) - query = urlencode(query) - url = urlunsplit((scheme, netloc, api_path + path.lstrip("/"), query, fragment)) - return url diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/misc.py ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/misc.py --- ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/misc.py 2021-09-18 15:20:06.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/misc.py 2021-12-05 15:20:37.000000000 +0000 @@ -50,6 +50,8 @@ UPLOAD_QUEUE_STATUSES = ('New', 'Unapproved', 'Accepted', 'Done', 'Rejected') +DOWNLOAD_BLOCKSIZE_DEFAULT = 8192 + _system_distribution_chain = [] @@ -286,7 +288,7 @@ return (url, None, None) -def download(src, dst, size=0): +def download(src, dst, size=0, *, blocksize=DOWNLOAD_BLOCKSIZE_DEFAULT): """ download/copy a file/url to local file src: str or Path @@ -295,11 +297,15 @@ Destination dir or filename size: int Size of source, if known + blocksize: int or None + Blocksize to use when downloading If the URL contains authentication data in the URL 'netloc', it will be stripped from the URL and passed to the requests library. This may throw a DownloadError. + + On success, this will return the dst as a Path object. """ src = str(src) parsedsrc = urlparse(src) @@ -318,11 +324,11 @@ if dst.exists(): if src.samefile(dst): Logger.info(f'Using existing file {dst}') - return + return dst Logger.info(f'Replacing existing file {dst}') Logger.info(f'Copying file {src} to {dst}') shutil.copyfile(src, dst) - return + return dst (src, username, password) = extract_authentication(src) auth = (username, password) if username or password else None @@ -332,7 +338,7 @@ try: with requests.get(src, stream=True, auth=auth) as fsrc, tmpdst.open('wb') as fdst: fsrc.raise_for_status() - _download(fsrc, fdst, size) + _download(fsrc, fdst, size, blocksize=blocksize) except requests.exceptions.HTTPError as e: if e.response is not None and e.response.status_code == 404: raise NotFoundError(f'URL {src} not found: {e}') @@ -343,9 +349,39 @@ except requests.exceptions.RequestException as e: raise DownloadError(e) shutil.move(tmpdst, dst) + return dst + +class _StderrProgressBar(object): + BAR_WIDTH_MIN = 40 + BAR_WIDTH_DEFAULT = 60 + + def __init__(self, max_width): + self.full_width = min(max_width, self.BAR_WIDTH_DEFAULT) + self.width = self.full_width - len('[] 99%') + self.show_progress = self.full_width >= self.BAR_WIDTH_MIN + + def update(self, progress, total): + if not self.show_progress: + return + pct = progress * 100 // total + pctstr = f'{pct:>3}%' + barlen = self.width * pct // 100 + barstr = '=' * barlen + barstr = barstr[:-1] + '>' + barstr = barstr.ljust(self.width) + fullstr = f'\r[{barstr}]{pctstr}' + sys.stderr.write(fullstr) + sys.stderr.flush() + + def finish(self): + if not self.show_progress: + return + sys.stderr.write('\n') + sys.stderr.flush() -def _download(fsrc, fdst, size): + +def _download(fsrc, fdst, size, *, blocksize): """ helper method to download src to dst using requests library. """ url = fsrc.url Logger.debug(f'Using URL: {url}') @@ -360,60 +396,62 @@ sizemb = ' (%0.3f MiB)' % (size / 1024.0 / 1024) if size else '' Logger.info(f'Downloading {filename} from {hostname}{sizemb}') - if not all((Logger.isEnabledFor(logging.INFO), sys.stderr.isatty(), size)): - shutil.copyfileobj(fsrc.raw, fdst) - return + # Don't show progress if: + # logging INFO is suppressed + # stderr isn't a tty + # we don't know the total file size + # the file is content-encoded (i.e. compressed) + show_progress = all((Logger.isEnabledFor(logging.INFO), + sys.stderr.isatty(), + size > 0, + 'Content-Encoding' not in fsrc.headers)) - blocksize = 4096 - XTRALEN = len('[] 99%') - downloaded = 0 - bar_width = 60 - term_width = os.get_terminal_size(sys.stderr.fileno())[0] - if term_width < bar_width + XTRALEN + 1: - bar_width = term_width - XTRALEN - 1 + terminal_width = 0 + if show_progress: + try: + terminal_width = os.get_terminal_size(sys.stderr.fileno()).columns + except Exception as e: + Logger.error(f'Error finding stderr width, suppressing progress bar: {e}') + progress_bar = _StderrProgressBar(max_width=terminal_width) + downloaded = 0 try: - while True: - block = fsrc.raw.read(blocksize) - if not block: - break + for block in fsrc.iter_content(blocksize): fdst.write(block) downloaded += len(block) - pct = float(downloaded) / size - bar = ('=' * int(pct * bar_width))[:-1] + '>' - fmt = '\r[{bar:<%d}]{pct:>3}%%\r' % bar_width - sys.stderr.write(fmt.format(bar=bar, pct=int(pct * 100))) - sys.stderr.flush() + progress_bar.update(downloaded, size) finally: - sys.stderr.write('\r' + ' ' * (term_width - 1) + '\r') - if downloaded < size: + progress_bar.finish() + if size and size > downloaded: Logger.error('Partial download: %0.3f MiB of %0.3f MiB' % (downloaded / 1024.0 / 1024, size / 1024.0 / 1024)) -def _download_text(src, binary): +def _download_text(src, binary, *, blocksize): with tempfile.TemporaryDirectory() as d: dst = Path(d) / 'dst' - download(src, dst) + download(src, dst, blocksize=blocksize) return dst.read_bytes() if binary else dst.read_text() -def download_text(src, mode=None): +def download_text(src, mode=None, *, blocksize=DOWNLOAD_BLOCKSIZE_DEFAULT): """ Return the text content of a downloaded file src: str or Path Source to copy from (file path or url) mode: str Deprecated, ignored unless a string that contains 'b' + blocksize: int or None + Blocksize to use when downloading Raises the same exceptions as download() Returns text content of downloaded file """ - return _download_text(src, binary='b' in (mode or '')) + return _download_text(src, binary='b' in (mode or ''), blocksize=blocksize) -def download_bytes(src): +def download_bytes(src, *, blocksize=DOWNLOAD_BLOCKSIZE_DEFAULT): """ Same as download_text() but returns bytes """ - return _download_text(src, binary=True) + return _download_text(src, binary=True, blocksize=blocksize) diff -Nru ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/pullpkg.py ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/pullpkg.py --- ubuntu-dev-tools-0.185~bpo20.04.1/ubuntutools/pullpkg.py 2021-02-03 13:24:39.000000000 +0000 +++ ubuntu-dev-tools-0.187~bpo20.04.1/ubuntutools/pullpkg.py 2021-12-05 15:20:37.000000000 +0000 @@ -26,6 +26,7 @@ import re import sys import errno +import subprocess from argparse import ArgumentParser @@ -401,7 +402,9 @@ if options['upload_queue']: # upload queue API is different/simpler - self.pull_upload_queue(pull, arch=options['arch'], **params) + self.pull_upload_queue(pull, arch=options['arch'], + download_only=options['download_only'], + **params) return # call implementation, and allow exceptions to flow up to caller @@ -458,27 +461,25 @@ Logger.error("Internal error: invalid pull value after parse_pull()") raise InvalidPullValueError("Invalid pull value '%s'" % pull) - def pull_upload_queue(self, pull, **params): - package = params['package'] - version = params['version'] - arch = params['arch'] - - if not params['series']: + def pull_upload_queue(self, pull, *, + package, version=None, arch=None, series=None, pocket=None, + status=None, download_only=None, **kwargs): + if not series: Logger.error("Using --upload-queue requires specifying series") return - series = Distribution('ubuntu').getSeries(params['series']) + series = Distribution('ubuntu').getSeries(series) queueparams = {'name': package} - if params['pocket']: - queueparams['pocket'] = params['pocket'] + if pocket: + queueparams['pocket'] = pocket - if params['status'] == 'all': + if status == 'all': queueparams['status'] = None queuetype = 'any' - elif params['status']: - queueparams['status'] = params['status'] - queuetype = params['status'] + elif status: + queueparams['status'] = status + queuetype = status else: queuetype = 'Unapproved' @@ -559,8 +560,23 @@ urls |= set(p.sourceFileUrls()) if not urls: Logger.error("No source files to download") + dscfile = None for url in urls: - download(url, os.getcwd()) + dst = download(url, os.getcwd()) + if dst.name.endswith('.dsc'): + dscfile = dst + if download_only: + Logger.debug("--download-only specified, not extracting") + elif not dscfile: + Logger.error("No source dsc file found, cannot extract") + else: + cmd = ['dpkg-source', '-x', dscfile.name] + Logger.debug(' '.join(cmd)) + result = subprocess.run(cmd, encoding='utf-8', + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode != 0: + Logger.error('Source unpack failed.') + Logger.debug(result.stdout) else: name = '.*' if pull == PULL_DEBS: