diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/bash_completion/pbuilder-dist ubuntu-dev-tools-0.201ubuntu2~22.04.1/bash_completion/pbuilder-dist --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/bash_completion/pbuilder-dist 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/bash_completion/pbuilder-dist 2023-06-05 02:04:11.000000000 +0000 @@ -36,7 +36,7 @@ for distro in $(ubuntu-distro-info --all; debian-distro-info --all) stable testing unstable; do for builder in pbuilder cowbuilder; do echo "$builder-$distro" - for arch in i386 amd64 armel armhf; do + for arch in i386 amd64 armhf; do echo "$builder-$distro-$arch" done done diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/check-mir ubuntu-dev-tools-0.201ubuntu2~22.04.1/check-mir --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/check-mir 2023-07-05 10:10:26.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/check-mir 2024-01-29 16:33:27.000000000 +0000 @@ -47,10 +47,41 @@ else: prefix = " * " + pkgname - try: + prov_packages = apt_cache.get_providing_packages(pkgname) + if pkgname in apt_cache: pkg = apt_cache[pkgname] - except KeyError: - print(prefix, "does not exist (pure virtual?)", file=sys.stderr) + + # If this is a virtual package, iterate through the binary packages that + # provide this, and ensure they are all in Main. Source packages in and of + # themselves cannot provide virtual packages, only binary packages can. + elif len(prov_packages) > 0: + supported, unsupported = [], [] + for pkg in prov_packages: + candidate = pkg.candidate + if candidate: + section = candidate.section + if section.startswith("universe") or section.startswith("multiverse"): + unsupported.append(pkg.name) + else: + supported.append(pkg.name) + + if len(supported) > 0: + msg = "is a virtual package, which is provided by the following " + msg += "candidates in Main: " + " ".join(supported) + print(prefix, msg) + elif len(unsupported) > 0: + msg = "is a virtual package, but is only provided by the " + msg += "following non-Main candidates: " + " ".join(unsupported) + print(prefix, msg, file=sys.stderr) + return False + else: + msg = "is a virtual package that exists but is not provided by " + msg += "package currently in the archive. Proceed with caution." + print(prefix, msg, file=sys.stderr) + return False + + else: + print(prefix, "does not exist", file=sys.stderr) return False section = pkg.candidate.section @@ -93,6 +124,13 @@ continue for or_group in apt.apt_pkg.parse_src_depends(control.section[field]): pkgname = or_group[0][0] + + # debhelper-compat is expected to be a build dependency of every + # package, so it is a red herring to display it in this report. + # (src:debhelper is in Ubuntu Main anyway) + if pkgname == "debhelper-compat": + continue + if not check_support(apt_cache, pkgname): # check non-preferred alternatives for altpkg in or_group[1:]: diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/debian/changelog ubuntu-dev-tools-0.201ubuntu2~22.04.1/debian/changelog --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/debian/changelog 2023-07-06 12:20:23.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/debian/changelog 2024-03-13 00:39:42.000000000 +0000 @@ -1,48 +1,132 @@ -ubuntu-dev-tools (0.193ubuntu4~22.04.2) jammy; urgency=medium +ubuntu-dev-tools (0.201ubuntu2~22.04.1) jammy; urgency=medium - * ubuntutools/misc.py: swap iter_content for raw stream with - "Accept-Encoding: identity" to fix .diff.gz downloads (LP: #2025748). This - fixes a regression caused by the backport of 0.193 that included this bug. + * Backport current ubuntu-dev-tools to jammy. LP: #2057716. - -- Robie Basak Thu, 06 Jul 2023 13:09:21 +0100 + -- Steve Langasek Tue, 12 Mar 2024 17:39:42 -0700 -ubuntu-dev-tools (0.193ubuntu4~22.04.1) jammy; urgency=medium +ubuntu-dev-tools (0.201ubuntu2) noble; urgency=medium - * Backport current ubuntu-dev-tools to jammy. LP: #2021910. - - this fixes compatibility with newer debootstrap which drops using - per-Ubuntu-series symlinks for the bootstrap script. LP: #2020530. + [ Steve Langasek ] + * ubuntu-build: support --batch with no package names to retry all + * ubuntu-build: in batch mode, print a count of packages retried + * ubuntu-build: make the --arch option top-level. + This gets rid of the fugly --arch2 option + * ubuntu-build: support retrying builds in other states that failed-to-build + * ubuntu-build: Handling of proposed vs release pocket default for ppas + * ubuntu-build: update manpage - -- Steve Langasek Tue, 30 May 2023 17:58:54 -0700 + -- Steve Langasek Tue, 12 Mar 2024 17:03:43 -0700 -ubuntu-dev-tools (0.193ubuntu4) mantic; urgency=medium +ubuntu-dev-tools (0.201ubuntu1) noble; urgency=medium - * Remove references to ftpmaster.internal. When this name is resolvable - but firewalled, syncpackage hangs; and these are tools for developers, - not for running in an automated context in the DCs where - ftpmaster.internal is reachable. + * Replace Depends on python3-launchpadlib with Depends on + python3-launchpadlib-desktop (LP: #2049217) - -- Steve Langasek Tue, 30 May 2023 10:02:43 -0700 + -- Chris Peterson Fri, 01 Mar 2024 14:08:07 -0800 -ubuntu-dev-tools (0.193ubuntu3) mantic; urgency=medium +ubuntu-dev-tools (0.201) unstable; urgency=medium - * Excise all references to cdbs (including in test cases) + * running-autopkgtests: fix packaging to make the script available + (LP: #2055466) - -- Steve Langasek Tue, 30 May 2023 09:45:12 -0700 + -- Chris Peterson Thu, 29 Feb 2024 11:09:14 -0800 -ubuntu-dev-tools (0.193ubuntu2) lunar; urgency=medium +ubuntu-dev-tools (0.200) unstable; urgency=medium - [ Vladimir Petko ] - * Fix a typo introduced in the last upload that made mk-sbuild fail - unconditionally. LP: #2017177. + [ Gianfranco Costamagna ] + * Team upload + + [ Chris Peterson ] + * Add support to see currently running autopkgtests (running-autopkgtests) + * running-autopkgtests: use f-strings + + [ Athos Ribeiro ] + * syncpackage: log LP authentication errors before halting. + + [ Ying-Chun Liu (PaulLiu) ] + * Drop qemu-debootstrap + qemu-debootstrap is deprecated for a while. In newer qemu release + the command is totally removed. We can use debootstrap directly. + Signed-off-by: Ying-Chun Liu (PaulLiu) + + [ Logan Rosen ] + * Don't rely on debootstrap for validating Ubuntu distro + + -- Gianfranco Costamagna Thu, 15 Feb 2024 17:53:48 +0100 + +ubuntu-dev-tools (0.199) unstable; urgency=medium + + [ Simon Quigley ] + * Add my name to Uploaders. + + [ Steve Langasek ] + * Introduce a pm-helper tool. + + -- Simon Quigley Mon, 29 Jan 2024 10:03:22 -0600 + +ubuntu-dev-tools (0.198) unstable; urgency=medium + + * In check-mir, ignore debhelper-compat when checking the build + dependencies. This is expected to be a build dependency of all packages, + so warning about it in any way is surely a red herring. + * Add proper support for virtual packages in check-mir, basing the + determination solely off of binary packages. This is not expected to be a + typical case. + + -- Simon Quigley Wed, 10 Jan 2024 20:04:02 -0600 - -- Steve Langasek Fri, 21 Apr 2023 02:23:08 +0000 +ubuntu-dev-tools (0.197) unstable; urgency=medium -ubuntu-dev-tools (0.193ubuntu1) lunar; urgency=medium + * Update the manpage for syncpackage to reflect the ability to sync + multiple packages at once. + * When using pull-*-source to grab a package which already has a defined + Vcs- field, display the exact same warning message `apt source` does. + -- Simon Quigley Tue, 03 Oct 2023 14:01:25 -0500 + +ubuntu-dev-tools (0.196) unstable; urgency=medium + + * Allow the user to sync multiple packages at one time (LP: #1756748). + + -- Simon Quigley Fri, 04 Aug 2023 14:37:59 -0500 + +ubuntu-dev-tools (0.195) unstable; urgency=medium + + * Add support for the non-free-firmware components in all tools already + referencing non-free. + + -- Simon Quigley Wed, 26 Jul 2023 13:03:31 -0500 + +ubuntu-dev-tools (0.194) unstable; urgency=medium + + [ Gianfranco Costamagna ] + * ubuntu-build: For some reasons, now you need to be authenticated before + trying to use the "PersonTeam" class features. + Do it at the begin instead of replicating the same code inside the + tool itself. + + [ Steve Langasek ] + * Remove references to deprecated + http://people.canonical.com/~ubuntu-archive. + * Remove references to architectures not supported in any active + Ubuntu release. + * Remove references to ftpmaster.internal. When this name is resolvable + but firewalled, syncpackage hangs; and these are tools for developers, + not for running in an automated context in the DCs where + ftpmaster.internal is reachable. + * Excise all references to cdbs (including in test cases) * Set apt preferences for the -proposed pocket in mk-sbuild so that it works as expected for lunar and forward. - -- Steve Langasek Wed, 12 Apr 2023 09:45:19 -0700 + [ Robie Basak ] + * ubuntutools/misc: swap iter_content for raw stream with "Accept-Encoding: + identity" to fix .diff.gz downloads (LP: #2025748). + + [ Vladimir Petko ] + * Fix a typo introduced in the last upload that made mk-sbuild fail + unconditionally. LP: #2017177. + + -- Gianfranco Costamagna Sat, 08 Jul 2023 08:42:05 +0200 ubuntu-dev-tools (0.193) unstable; urgency=medium diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/debian/control ubuntu-dev-tools-0.201ubuntu2~22.04.1/debian/control --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/debian/control 2023-07-06 12:20:17.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/debian/control 2024-03-13 00:01:28.000000000 +0000 @@ -1,11 +1,13 @@ Source: ubuntu-dev-tools Section: devel Priority: optional -Maintainer: Ubuntu Developers +Maintainer: Ubuntu Developers +XSBC-Original-Maintainer: Ubuntu Developers Uploaders: Benjamin Drung , Stefano Rivera , Mattia Rizzolo , + Simon Quigley , Build-Depends: black , dctrl-tools, @@ -20,11 +22,12 @@ pylint , python3-all, python3-apt, + python3-dateutil, python3-debian, python3-debianbts, python3-distro-info, python3-httplib2, - python3-launchpadlib, + python3-launchpadlib-desktop, python3-pytest, python3-requests , python3-setuptools, @@ -51,7 +54,7 @@ python3-debianbts, python3-distro-info, python3-httplib2, - python3-launchpadlib, + python3-launchpadlib-desktop, python3-lazr.restfulclient, python3-ubuntutools (= ${binary:Version}), sensible-utils, @@ -114,6 +117,8 @@ - requestsync - files a sync request with Debian changelog and rationale. - reverse-depends - find the reverse dependencies (or build dependencies) of a package. + - running-autopkgtests - lists the currently running and/or queued + autopkgtests on the Ubuntu autopkgtest infrastructure - seeded-in-ubuntu - query if a package is safe to upload during a freeze. - setup-packaging-environment - assistant to get an Ubuntu installation ready for packaging work. @@ -132,10 +137,11 @@ Architecture: all Section: python Depends: + python3-dateutil, python3-debian, python3-distro-info, python3-httplib2, - python3-launchpadlib, + python3-launchpadlib-desktop, python3-lazr.restfulclient, python3-requests, sensible-utils, diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/pbuilder-dist.1 ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/pbuilder-dist.1 --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/pbuilder-dist.1 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/pbuilder-dist.1 2023-06-05 02:04:11.000000000 +0000 @@ -38,7 +38,7 @@ architecture. For some architecture pairs (e.g. i386 on an amd64 install), the chroot will be created natively. -For others (e.g. armel on an i386 install), qemu\-user\-static will be +For others (e.g. arm64 on an amd64 install), qemu\-user\-static will be used. Note that some combinations (e.g. amd64 on an i386 install) require special separate kernel handling, and may break in unexpected ways. diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/pm-helper.1 ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/pm-helper.1 --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/pm-helper.1 1970-01-01 00:00:00.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/pm-helper.1 2024-01-29 16:33:27.000000000 +0000 @@ -0,0 +1,44 @@ +.\" Copyright (C) 2023, Canonical Ltd. +.\" +.\" This program is free software; you can redistribute it and/or +.\" modify it under the terms of the GNU General Public License, version 3. +.\" +.\" This program is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +.\" General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with this program. If not, see . +.TH pm\-helper 1 "June 2023" ubuntu\-dev\-tools + +.SH NAME +pm\-helper \- helper to guide a developer through proposed\-migration work + +.SH SYNOPSIS +.B pm\-helper \fR[\fIoptions\fR] [\fIpackage\fR] + +.SH DESCRIPTION +Claim a package from proposed\-migration to work on and get additional +information (such as the state of the package in Debian) that may be helpful +in unblocking it. +.PP +This tool is incomplete and under development. + +.SH OPTIONS +.TP +.B \-l \fIINSTANCE\fR, \fB\-\-launchpad\fR=\fIINSTANCE\fR +Use the specified instance of Launchpad (e.g. "staging"), instead of +the default of "production". +.TP +.B \-v\fR, \fB--verbose\fR +be more verbose +.TP +\fB\-h\fR, \fB\-\-help\fR +Display a help message and exit + +.SH AUTHORS +\fBpm\-helper\fR and this manpage were written by Steve Langasek +. +.PP +Both are released under the GPLv3 license. diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/syncpackage.1 ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/syncpackage.1 --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/syncpackage.1 2023-07-05 10:10:26.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/syncpackage.1 2023-11-09 09:45:39.000000000 +0000 @@ -4,11 +4,11 @@ .\" .SH SYNOPSIS .B syncpackage -[\fIoptions\fR] \fI<.dsc URL/path or package name>\fR +[\fIoptions\fR] \fI<.dsc URL/path or package name(s)>\fR .\" .SH DESCRIPTION -\fBsyncpackage\fR causes a source package to be copied from Debian to -Ubuntu. +\fBsyncpackage\fR causes one or more source package(s) to be copied from Debian +to Ubuntu. .PP \fBsyncpackage\fR allows you to upload files with the same checksums of the Debian ones, as the common script used by Ubuntu archive administrators does, diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/ubuntu-build.1 ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/ubuntu-build.1 --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/doc/ubuntu-build.1 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/doc/ubuntu-build.1 2024-03-13 00:01:24.000000000 +0000 @@ -1,9 +1,14 @@ -.TH UBUNTU-BUILD "1" "June 2010" "ubuntu-dev-tools" +.TH UBUNTU-BUILD "1" "Mar 2024" "ubuntu-dev-tools" .SH NAME ubuntu-build \- command-line interface to Launchpad build operations .SH SYNOPSIS -.B ubuntu-build +.nf +\fBubuntu-build\fR +\fBubuntu-build\fR --batch [--retry] [--rescore \fIPRIORITY\fR] [--arch \fIARCH\fR [...]] + [--series \fISERIES\fR] [--state \fIBUILD-STATE\fR] + [-A \fIARCHIVE\fR] [pkg]... +.fi .SH DESCRIPTION \fBubuntu-build\fR provides a command line interface to the Launchpad build @@ -38,8 +43,7 @@ \fB\-a\fR ARCHITECTURE, \fB\-\-arch\fR=\fIARCHITECTURE\fR Rebuild or rescore a specific architecture. Valid architectures are: -armel, armhf, arm64, amd64, hppa, i386, ia64, -lpia, powerpc, ppc64el, riscv64, s390x, sparc. +armhf, arm64, amd64, i386, powerpc, ppc64el, riscv64, s390x. .TP Batch processing: .IP @@ -59,15 +63,16 @@ \fB\-\-rescore\fR=\fIPRIORITY\fR Rescore builds to . .IP -\fB\-\-arch2\fR=\fIARCHITECTURE\fR +\fB\-\-arch\fR=\fIARCHITECTURE\fR Affect only 'architecture' (can be used several times). Valid architectures are: -armel, armhf, arm64, amd64, hppa, i386, ia64, -lpia, powerpc, ppc64el, riscv64, s390x, sparc. +arm64, amd64, i386, powerpc, ppc64el, riscv64, s390x. +.IP +\fB\-A=\fIARCHIVE\fR +Act on the named archive (ppa) instead of on the main Ubuntu archive. .SH AUTHORS \fBubuntu-build\fR was written by Martin Pitt , and this manual page was written by Jonathan Patrick Davies . .PP -Both are released under the terms of the GNU General Public License, version 3 -or (at your option) any later version. +Both are released under the terms of the GNU General Public License, version 3. diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/mk-sbuild ubuntu-dev-tools-0.201ubuntu2~22.04.1/mk-sbuild --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/mk-sbuild 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/mk-sbuild 2024-03-12 18:52:03.000000000 +0000 @@ -166,7 +166,7 @@ --arch) CHROOT_ARCH="$2" case $2 in - armel|armhf|i386|lpia) + armhf|i386) if [ -z "$personality" ]; then personality="linux32" fi @@ -667,7 +667,7 @@ DEBOOTSTRAP_MIRROR="http://deb.debian.org/debian" fi if [ -z "$COMPONENTS" ]; then - COMPONENTS="main non-free contrib" + COMPONENTS="main non-free non-free-firmware contrib" fi if [ -z "$SOURCES_PROPOSED_SUITE" ]; then SOURCES_PROPOSED_SUITE="RELEASE-proposed-updates" @@ -750,12 +750,12 @@ if [ "$CHROOT_ARCH" != "$HOST_ARCH" ] ; then case "$CHROOT_ARCH-$HOST_ARCH" in # Sometimes we don't need qemu - amd64-i386|amd64-lpia|armel-armhf|armhf-armel|arm64-armel|arm64-armhf|armel-arm64|armhf-arm64|i386-amd64|i386-lpia|lpia-i386|powerpc-ppc64|ppc64-powerpc|sparc-sparc64|sparc64-sparc) + amd64-i386|arm64-armhf|armhf-arm64|i386-amd64|powerpc-ppc64|ppc64-powerpc) ;; # Sometimes we do *) - DEBOOTSTRAP_COMMAND=qemu-debootstrap - if ! which "$DEBOOTSTRAP_COMMAND"; then + DEBOOTSTRAP_COMMAND=debootstrap + if ! which "qemu-x86_64-static"; then sudo apt-get install qemu-user-static fi ;; diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/pbuilder-dist ubuntu-dev-tools-0.201ubuntu2~22.04.1/pbuilder-dist --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/pbuilder-dist 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/pbuilder-dist 2024-03-12 18:52:03.000000000 +0000 @@ -95,7 +95,11 @@ # Builder self.builder = builder - self._debian_distros = DebianDistroInfo().all + ["stable", "testing", "unstable"] + # Distro info + self.debian_distro_info = DebianDistroInfo() + self.ubuntu_distro_info = UbuntuDistroInfo() + + self._debian_distros = self.debian_distro_info.all + ["stable", "testing", "unstable"] # Ensure that the used builder is installed paths = set(os.environ["PATH"].split(":")) @@ -151,8 +155,9 @@ if not os.path.isfile(os.path.join("/usr/share/debootstrap/scripts/", distro)): if os.path.isdir("/usr/share/debootstrap/scripts/"): # Debian experimental doesn't have a debootstrap file but - # should work nevertheless. - if distro not in self._debian_distros: + # should work nevertheless. Ubuntu releases automatically use + # the gutsy script as of debootstrap 1.0.128+nmu2ubuntu1.1. + if distro not in (self._debian_distros + self.ubuntu_distro_info.all): question = ( f'Warning: Unknown distribution "{distro}". ' "Do you want to continue" ) @@ -270,7 +275,7 @@ mirror = os.environ.get("MIRRORSITE", config.get_value("DEBIAN_MIRROR")) components = "main" if self.extra_components: - components += " contrib non-free" + components += " contrib non-free non-free-firmware" else: mirror = os.environ.get("MIRRORSITE", config.get_value("UBUNTU_MIRROR")) if self.build_architecture not in ("amd64", "i386"): @@ -288,23 +293,22 @@ othermirrors.append(repo) if self.target_distro in self._debian_distros: - debian_info = DebianDistroInfo() try: - codename = debian_info.codename(self.target_distro, default=self.target_distro) + codename = self.debian_distro_info.codename(self.target_distro, default=self.target_distro) except DistroDataOutdated as error: Logger.warning(error) - if codename in (debian_info.devel(), "experimental"): + if codename in (self.debian_distro_info.devel(), "experimental"): self.enable_security = False self.enable_updates = False self.enable_proposed = False - elif codename in (debian_info.testing(), "testing"): + elif codename in (self.debian_distro_info.testing(), "testing"): self.enable_updates = False if self.enable_security: pocket = "-security" with suppress(ValueError): # before bullseye (version 11) security suite is /updates - if float(debian_info.version(codename)) < 11.0: + if float(self.debian_distro_info.version(codename)) < 11.0: pocket = "/updates" othermirrors.append( f"deb {config.get_value('DEBSEC_MIRROR')}" @@ -322,7 +326,7 @@ aptcache = os.path.join(self.base, "aptcache", "debian") else: try: - dev_release = self.target_distro == UbuntuDistroInfo().devel() + dev_release = self.target_distro == self.ubuntu_distro_info.devel() except DistroDataOutdated as error: Logger.warning(error) dev_release = True @@ -487,22 +491,12 @@ requested_arch, ) not in [ ("amd64", "i386"), - ("amd64", "lpia"), - ("arm", "armel"), - ("armel", "arm"), - ("armel", "armhf"), - ("armhf", "armel"), ("arm64", "arm"), ("arm64", "armhf"), - ("arm64", "armel"), - ("i386", "lpia"), - ("lpia", "i386"), ("powerpc", "ppc64"), ("ppc64", "powerpc"), - ("sparc", "sparc64"), - ("sparc64", "sparc"), ]: - args += ["--debootstrap", "qemu-debootstrap"] + args += ["--debootstrap", "debootstrap"] if "mainonly" in sys.argv or "--main-only" in sys.argv: app.extra_components = False diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/pm-helper ubuntu-dev-tools-0.201ubuntu2~22.04.1/pm-helper --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/pm-helper 1970-01-01 00:00:00.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/pm-helper 2024-01-29 16:33:27.000000000 +0000 @@ -0,0 +1,149 @@ +#!/usr/bin/python3 +# Find the next thing to work on for proposed-migration +# Copyright (C) 2023 Canonical Ltd. +# Author: Steve Langasek + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License, version 3. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import lzma +from argparse import ArgumentParser +import sys +import webbrowser +import yaml + +from launchpadlib.launchpad import Launchpad + +from ubuntutools.utils import get_url + + +# proposed-migration is only concerned with the devel series; unlike other +# tools, don't make this configurable +excuses_url = 'https://ubuntu-archive-team.ubuntu.com/proposed-migration/' \ + + 'update_excuses.yaml.xz' + + +def get_proposed_version(excuses, package): + for k in excuses['sources']: + if k['source'] == package: + return k.get('new-version') + return None + + +def claim_excuses_bug(launchpad, bug, package): + print("LP: #%d: %s" % (bug.id, bug.title)) + ubuntu = launchpad.distributions['ubuntu'] + series = ubuntu.current_series.fullseriesname + + for task in bug.bug_tasks: + # targeting to a series doesn't make the default task disappear, + # it just makes it useless + if task.bug_target_name == "%s (%s)" % (package, series): + our_task = task + break + elif task.bug_target_name == "%s (Ubuntu)" % package: + our_task = task + + if our_task.assignee == launchpad.me: + print("Bug already assigned to you.") + return True + elif our_task.assignee: + print("Currently assigned to %s" % our_task.assignee.name) + + print('''Do you want to claim this bug? [yN] ''', end="") + sys.stdout.flush() + response = sys.stdin.readline() + if response.strip().lower().startswith('y'): + our_task.assignee = launchpad.me + our_task.lp_save() + return True + + return False + + +def create_excuses_bug(launchpad, package, version): + print("Will open a new bug") + bug = launchpad.bugs.createBug( + title = 'proposed-migration for %s %s' % (package, version), + tags = ('update-excuse'), + target = 'https://api.launchpad.net/devel/ubuntu/+source/%s' % package, + description = '%s %s is stuck in -proposed.' % (package, version) + ) + + task = bug.bug_tasks[0] + task.assignee = launchpad.me + task.lp_save() + + print("Opening %s in browser" % bug.web_link) + webbrowser.open(bug.web_link) + return bug + + +def has_excuses_bugs(launchpad, package): + ubuntu = launchpad.distributions['ubuntu'] + pkg = ubuntu.getSourcePackage(name=package) + if not pkg: + raise ValueError(f"No such source package: {package}") + + tasks = pkg.searchTasks(tags=['update-excuse'], order_by=['id']) + + bugs = [task.bug for task in tasks] + if not bugs: + return False + + if len(bugs) == 1: + print("There is 1 open update-excuse bug against %s" % package) + else: + print("There are %d open update-excuse bugs against %s" \ + % (len(bugs), package)) + + for bug in bugs: + if claim_excuses_bug(launchpad, bug, package): + return True + + return True + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "-l", "--launchpad", dest="launchpad_instance", default="production") + parser.add_argument( + "-v", "--verbose", default=False, action="store_true", + help="be more verbose") + parser.add_argument( + 'package', nargs='?', help="act on this package only") + args = parser.parse_args() + + args.launchpad = Launchpad.login_with( + "pm-helper", args.launchpad_instance, version="devel") + + f = get_url(excuses_url, False) + with lzma.open(f) as lzma_f: + excuses = yaml.load(lzma_f, Loader=yaml.CSafeLoader) + + if args.package: + try: + if not has_excuses_bugs(args.launchpad, args.package): + proposed_version = get_proposed_version(excuses, args.package) + if not proposed_version: + print("Package %s not found in -proposed." % args.package) + sys.exit(1) + create_excuses_bug(args.launchpad, args.package, + proposed_version) + except ValueError as e: + sys.stderr.write(f"{e}\n") + else: + pass # for now + + +if __name__ == '__main__': + sys.exit(main()) diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/requirements.txt ubuntu-dev-tools-0.201ubuntu2~22.04.1/requirements.txt --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/requirements.txt 2023-07-05 10:10:26.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/requirements.txt 2024-01-29 16:33:27.000000000 +0000 @@ -1,5 +1,6 @@ python-debian python-debianbts +dateutil distro-info httplib2 launchpadlib diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/running-autopkgtests ubuntu-dev-tools-0.201ubuntu2~22.04.1/running-autopkgtests --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/running-autopkgtests 1970-01-01 00:00:00.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/running-autopkgtests 2024-03-12 18:52:03.000000000 +0000 @@ -0,0 +1,69 @@ +#!/usr/bin/python3 +# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*- + +# Authors: +# Andy P. Whitcroft +# Christian Ehrhardt + +"""Dumps a list of currently running tests in Autopkgtest""" + +__example__ = """ +Display first listed test running on amd64 hardware: + $ running-autopkgtests | grep amd64 | head -n1 + R 0:01:40 systemd-upstream - focal amd64 upstream-systemd-ci/systemd-ci - ['CFLAGS=-O0', 'DEB_BUILD_PROFILES=noudeb', 'TEST_UPSTREAM=1', 'CONFFLAGS_UPSTREAM=--werror -Dslow-tests=true', 'UPSTREAM_PULL_REQUEST=23153', 'GITHUB_STATUSES_URL=https://api.github.com/repos/systemd/systemd/statuses/cfb0935923dff8050315b5dd22ce8ab06461ff0e'] +""" + +import sys +from argparse import ArgumentParser, RawDescriptionHelpFormatter + +from ubuntutools.running_autopkgtests import get_queued, get_running + + +def parse_args(): + description = ( + "Dumps a list of currently running and queued tests in Autopkgtest. " + "Pass --running to only see running tests, or --queued to only see " + "queued tests. Passing both will print both, which is the default behavior. " + ) + + parser = ArgumentParser( + prog="running-autopkgtests", + description=description, + epilog=f"example: {__example__}", + formatter_class=RawDescriptionHelpFormatter, + ) + parser.add_argument( + "-r", + "--running", + action="store_true", + help="Print runnning autopkgtests (default: true)", + ) + parser.add_argument( + "-q", + "--queued", + action="store_true", + help="Print queued autopkgtests (default: true)", + ) + + options = parser.parse_args() + + # If neither flag was specified, default to both not neither + if not options.running and not options.queued: + options.running = True + options.queued = True + + return options + + +def main() -> int: + args = parse_args() + if args.running: + print(get_running()) + if args.queued: + print(get_queued()) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/setup.py ubuntu-dev-tools-0.201ubuntu2~22.04.1/setup.py --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/setup.py 2023-07-05 10:10:26.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/setup.py 2024-03-12 18:52:03.000000000 +0000 @@ -64,6 +64,7 @@ "requestbackport", "requestsync", "reverse-depends", + "running-autopkgtests", "seeded-in-ubuntu", "setup-packaging-environment", "sponsor-patch", diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/syncpackage ubuntu-dev-tools-0.201ubuntu2~22.04.1/syncpackage --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/syncpackage 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/syncpackage 2024-03-12 18:52:03.000000000 +0000 @@ -324,7 +324,7 @@ if component is None: component = debian_srcpkg.getComponent() - assert component in ("main", "contrib", "non-free") + assert component in ("main", "contrib", "non-free", "non-free-firmware") return DebianSourcePackage(package, version.full_version, component, mirrors=mirrors) @@ -452,7 +452,7 @@ blacklisted = "ALWAYS" # Old blacklist: - url = "http://people.canonical.com/~ubuntu-archive/sync-blacklist.txt" + url = "https://ubuntu-archive-team.ubuntu.com/sync-blacklist.txt" with urllib.request.urlopen(url) as f: applicable_lines = [] for line in f: @@ -504,7 +504,7 @@ def parse(): """Parse given command-line parameters.""" - usage = "%(prog)s [options] <.dsc URL/path or package name>" + usage = "%(prog)s [options] <.dsc URL/path or package name(s)>" epilog = f"See {os.path.basename(sys.argv[0])}(1) for more info." parser = argparse.ArgumentParser(usage=usage, epilog=epilog) @@ -602,7 +602,7 @@ metavar="UBUNTU_MIRROR", help=f"Preferred Ubuntu mirror (default: {UDTConfig.defaults['UBUNTU_MIRROR']})", ) - parser.add_argument("package", help=argparse.SUPPRESS) + parser.add_argument("package", nargs="*", help=argparse.SUPPRESS) args = parser.parse_args() if args.fakesync: @@ -613,10 +613,10 @@ except TypeError: parser.error("Invalid bug number(s) specified.") - if args.component not in (None, "main", "contrib", "non-free"): + if args.component not in (None, "main", "contrib", "non-free", "non-free-firmware"): parser.error( f"{args.component} is not a valid Debian component. " - f"It should be one of main, contrib, or non-free." + f"It should be one of main, contrib, non-free, or non-free-firmware." ) if args.lp and args.uploader_name: @@ -627,8 +627,9 @@ # ignored with args.lp, and do not require warnings. if args.lp: - if args.package.endswith(".dsc"): - parser.error(".dsc files can only be synced using --no-lp.") + for package in args.package: + if package.endswith(".dsc"): + parser.error(".dsc files can only be synced using --no-lp.") return args @@ -651,14 +652,15 @@ if args.lpinstance is None: args.lpinstance = config.get_value("LPINSTANCE") + # devel for copyPackage and changelogUrl + kwargs = {"service": args.lpinstance, "api_version": "devel"} try: - # devel for copyPackage and changelogUrl - kwargs = {"service": args.lpinstance, "api_version": "devel"} if args.lp and not args.simulate: Launchpad.login(**kwargs) else: Launchpad.login_anonymously(**kwargs) - except IOError: + except IOError as e: + Logger.error("Could not authenticate to LP: %s", str(e)) sys.exit(1) if args.release is None: @@ -701,75 +703,76 @@ elif args.uploader_email is None: args.uploader_email = ubu_email(export=False)[1] - src_pkg = fetch_source_pkg( - args.package, - args.distribution, - args.debian_version, - args.component, - args.release, - args.debian_mirror, - ) - - blacklisted, comments = is_blacklisted(src_pkg.source) - blacklist_fail = False - if blacklisted: - messages = [] - - if blacklisted == "CURRENT": - Logger.debug( - "Source package %s is temporarily blacklisted " - "(blacklisted_current). " - "Ubuntu ignores these for now. " - "See also LP: #841372", - src_pkg.source, - ) - else: - if args.fakesync: - messages += ["Doing a fakesync, overriding blacklist."] + for package in args.package: + src_pkg = fetch_source_pkg( + package, + args.distribution, + args.debian_version, + args.component, + args.release, + args.debian_mirror, + ) + + blacklisted, comments = is_blacklisted(src_pkg.source) + blacklist_fail = False + if blacklisted: + messages = [] + + if blacklisted == "CURRENT": + Logger.debug( + "Source package %s is temporarily blacklisted " + "(blacklisted_current). " + "Ubuntu ignores these for now. " + "See also LP: #841372", + src_pkg.source, + ) else: - blacklist_fail = True - messages += [ - "If this package needs a fakesync, use --fakesync", - "If you think this package shouldn't be " - "blacklisted, please file a bug explaining your " - "reasoning and subscribe ~ubuntu-archive.", - ] + if args.fakesync: + messages += ["Doing a fakesync, overriding blacklist."] + else: + blacklist_fail = True + messages += [ + "If this package needs a fakesync, use --fakesync", + "If you think this package shouldn't be " + "blacklisted, please file a bug explaining your " + "reasoning and subscribe ~ubuntu-archive.", + ] + + if blacklist_fail: + Logger.error("Source package %s is blacklisted.", src_pkg.source) + elif blacklisted == "ALWAYS": + Logger.info("Source package %s is blacklisted.", src_pkg.source) + if messages: + for message in messages: + for line in textwrap.wrap(message): + Logger.info(line) + + if comments: + Logger.info("Blacklist Comments:") + for comment in comments: + for line in textwrap.wrap(comment): + Logger.info(" %s", line) if blacklist_fail: - Logger.error("Source package %s is blacklisted.", src_pkg.source) - elif blacklisted == "ALWAYS": - Logger.info("Source package %s is blacklisted.", src_pkg.source) - if messages: - for message in messages: - for line in textwrap.wrap(message): - Logger.info(line) - - if comments: - Logger.info("Blacklist Comments:") - for comment in comments: - for line in textwrap.wrap(comment): - Logger.info(" %s", line) - - if blacklist_fail: - sys.exit(1) + sys.exit(1) - if args.lp: - copy(src_pkg, args.release, args.bugs, sponsoree, args.simulate, args.force) - else: - os.environ["DEB_VENDOR"] = "Ubuntu" - sync_dsc( - src_pkg, - args.distribution, - args.release, - args.uploader_name, - args.uploader_email, - args.bugs, - args.ubuntu_mirror, - args.keyid, - args.simulate, - args.force, - args.fakesync, - ) + if args.lp: + copy(src_pkg, args.release, args.bugs, sponsoree, args.simulate, args.force) + else: + os.environ["DEB_VENDOR"] = "Ubuntu" + sync_dsc( + src_pkg, + args.distribution, + args.release, + args.uploader_name, + args.uploader_email, + args.bugs, + args.ubuntu_mirror, + args.keyid, + args.simulate, + args.force, + args.fakesync, + ) if __name__ == "__main__": diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntu-build ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntu-build --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntu-build 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntu-build 2024-03-13 00:01:24.000000000 +0000 @@ -2,16 +2,16 @@ # # ubuntu-build - command line interface for Launchpad buildd operations. # -# Copyright (C) 2007 Canonical Ltd. +# Copyright (C) 2007-2024 Canonical Ltd. # Authors: # - Martin Pitt # - Jonathan Davies # - Michael Bienia +# - Steve Langasek # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -29,19 +29,63 @@ import sys from launchpadlib.credentials import TokenAuthorizationException +from launchpadlib.launchpad import Launchpad +import lazr.restfulclient.errors from ubuntutools import getLogger -from ubuntutools.lp.lpapicache import Distribution, Launchpad, PersonTeam -from ubuntutools.lp.udtexceptions import ( - PackageNotFoundException, - PocketDoesNotExistError, - SeriesNotFoundException, -) +from ubuntutools.lp.udtexceptions import PocketDoesNotExistError from ubuntutools.misc import split_release_pocket Logger = getLogger() +def getBuildStates(pkg, archs): + res = [] + + for build in pkg.getBuilds(): + if build.arch_tag in archs: + res.append(f" {build.arch_tag}: {build.buildstate}") + msg = "\n".join(res) + return f"Build state(s) for '{pkg.source_package_name}':\n{msg}" + +def rescoreBuilds(pkg, archs, score): + res = [] + + for build in pkg.getBuilds(): + arch = build.arch_tag + if arch in archs: + if not build.can_be_rescored: + continue + try: + build.rescore(score=score) + res.append(f" {arch}: done") + except lazr.restfulclient.errors.Unauthorized: + Logger.error( + "You don't have the permissions to rescore builds. Ignoring your rescore request." + ) + return None + except lazr.restfulclient.errors.BadRequest: + Logger.info("Cannot rescore build of %s on %s.", + build.source_package_name, arch) + res.append(f" {arch}: failed") + + msg = "\n".join(res) + return f"Rescoring builds of '{pkg.source_package_name}' to {score}:\n{msg}" + +def retryBuilds(pkg, archs): + res = [] + for build in pkg.getBuilds(): + arch = build.arch_tag + if arch in archs: + try: + build.retry() + res.append(f" {arch}: done") + except lazr.restfulclient.errors.BadRequest: + res.append(f" {arch}: failed") + msg = "\n".join(res) + return f"Retrying builds of '{pkg.source_package_name}':\n{msg}" + + def main(): # Usage. usage = "%(prog)s \n\n" @@ -51,31 +95,21 @@ # Valid architectures. valid_archs = set( [ - "armel", "armhf", "arm64", "amd64", - "hppa", "i386", - "ia64", - "lpia", "powerpc", "ppc64el", "riscv64", "s390x", - "sparc", ] ) # Prepare our option parser. parser = argparse.ArgumentParser(usage=usage) - # Retry options - retry_rescore_options = parser.add_argument_group( - "Retry and rescore options", - "These options may only be used with the 'retry' and 'rescore' operations.", - ) - retry_rescore_options.add_argument( + parser.add_argument( "-a", "--arch", action="append", @@ -84,6 +118,9 @@ f"include: {', '.join(valid_archs)}.", ) + parser.add_argument("-A", "--archive", help="operate on ARCHIVE", + default="ubuntu") + # Batch processing options batch_options = parser.add_argument_group( "Batch processing", @@ -111,18 +148,35 @@ help="Rescore builds to .", ) batch_options.add_argument( - "--arch2", - action="append", - dest="architecture", - help=f"Affect only 'architecture' (can be used several times)." - f" Valid architectures are: {', '.join(valid_archs)}.", + "--state", action="store", dest="state", + help="Act on builds that are in the specified state", ) - parser.add_argument("packages", metavar="package", nargs="+", help=argparse.SUPPRESS) + + parser.add_argument("packages", metavar="package", nargs="*", help=argparse.SUPPRESS) # Parse our options. args = parser.parse_args() - if not args.batch: + launchpad = Launchpad.login_with("ubuntu-dev-tools", "production", + version="devel") + me = launchpad.me + + ubuntu = launchpad.distributions['ubuntu'] + + if args.batch: + release = args.series + if not release: + # ppas don't have a proposed pocket so just use the release pocket; + # but for the main archive we default to -proposed + release = ubuntu.getDevelopmentSeries()[0].name + if args.archive == 'ubuntu': + release = release + "-proposed" + try: + (release, pocket) = split_release_pocket(release) + except PocketDoesNotExistError as error: + Logger.error(error) + sys.exit(1) + else: # Check we have the correct number of arguments. if len(args.packages) < 3: parser.error("Incorrect number of arguments.") @@ -135,6 +189,14 @@ parser.print_help() sys.exit(1) + archive = launchpad.archives.getByReference(reference=args.archive) + try: + distroseries = ubuntu.getSeries(name_or_version=release) + except lazr.restfulclient.errors.NotFound as error: + Logger.error(error) + sys.exit(1) + + if not args.batch: # Check our operation. if operation not in ("rescore", "retry", "status"): Logger.error("Invalid operation: %s.", operation) @@ -158,44 +220,37 @@ Logger.error(error) sys.exit(1) - try: - # Will fail here if we have no credentials, bail out - Launchpad.login() - except TokenAuthorizationException: - sys.exit(1) - - # Get the ubuntu archive - ubuntu_archive = Distribution("ubuntu").getArchive() # Get list of published sources for package in question. try: - sources = ubuntu_archive.getSourcePackage(package, release, pocket) - distroseries = Distribution("ubuntu").getSeries(release) - except (SeriesNotFoundException, PackageNotFoundException) as error: - Logger.error(error) + sources = archive.getPublishedSources( + distro_series=distroseries, + exact_match=True, + pocket=pocket, + source_name=package, + status='Published')[0] + except IndexError as error: + Logger.error("No publication found for package %s", package) sys.exit(1) # Get list of builds for that package. builds = sources.getBuilds() # Find out the version and component in given release. - version = sources.getVersion() - component = sources.getComponent() + version = sources.source_package_version + component = sources.component_name # Operations that are remaining may only be done by Ubuntu developers # (retry) or buildd admins (rescore). Check if the proper permissions # are in place. - me = PersonTeam.me - if operation == "rescore": - necessary_privs = me.isLpTeamMember("launchpad-buildd-admins") if operation == "retry": - necessary_privs = me.canUploadPackage( - ubuntu_archive, - distroseries, - sources.getPackageName(), - sources.getComponent(), - pocket=pocket, + necessary_privs = archive.checkUpload( + component=sources.getComponent(), + distroseries=distroseries, + person=launchpad.me, + pocket=pocket, + sourcepackagename=sources.getPackageName(), ) - if operation in ("rescore", "retry") and not necessary_privs: + if operation == "retry" and not necessary_privs: Logger.error( "You cannot perform the %s operation on a %s package as you" " do not have the permissions to do this action.", @@ -229,7 +284,13 @@ # FIXME: make priority an option priority = 5000 Logger.info("Rescoring build %s to %d...", build.arch_tag, priority) - build.rescore(score=priority) + try: + build.rescore(score=priority) + except lazr.restfulclient.errors.Unauthorized: + Logger.error( + "You don't have the permissions to rescore builds. Ignoring your rescore request." + ) + break else: Logger.info("Cannot rescore build on %s.", build.arch_tag) if operation == "retry": @@ -258,65 +319,131 @@ # filter out duplicate and invalid architectures archs.intersection_update(valid_archs) - release = args.series - if not release: - release = Distribution("ubuntu").getDevelopmentSeries().name + "-proposed" - try: - (release, pocket) = split_release_pocket(release) - except PocketDoesNotExistError as error: - Logger.error(error) - sys.exit(1) + if not args.packages: + retry_count = 0 + can_rescore = True + + if not args.state: + if args.retry: + args.state='Failed to build' + elif args.priority: + args.state='Needs building' + # there is no equivalent to series.getBuildRecords() for a ppa. + # however, we don't want to have to traverse all build records for + # all series when working on the main archive, so we use + # series.getBuildRecords() for ubuntu and handle ppas separately + series = ubuntu.getSeries(name_or_version=release) + if args.archive == 'ubuntu': + builds = series.getBuildRecords(build_state=args.state, + pocket=pocket) + else: + builds = [] + for build in archive.getBuildRecords(build_state=args.state, + pocket=pocket): + if not build.current_source_publication: + continue + if build.current_source_publication.distro_series==series: + builds.append(build) + for build in builds: + if build.arch_tag not in archs: + continue + if not build.current_source_publication: + continue + # fixme: refactor + # Check permissions (part 2): check upload permissions for the + # source package + can_retry = args.retry and archive.checkUpload( + component=build.current_source_publication.component_name, + distroseries=series, + person=launchpad.me, + pocket=pocket, + sourcepackagename=build.source_package_name, + ) + if args.retry and not can_retry: + Logger.error( + "You don't have the permissions to retry the " + "build of '%s', skipping.", + build.source_package_name + ) + continue + Logger.info( + "The source version for '%s' in '%s' (%s) is: %s", + build.source_package_name, + release, + pocket, + build.source_package_version + ) - ubuntu_archive = Distribution("ubuntu").getArchive() - try: - distroseries = Distribution("ubuntu").getSeries(release) - except SeriesNotFoundException as error: - Logger.error(error) - sys.exit(1) - me = PersonTeam.me + if args.retry and build.can_be_retried: + Logger.info("Retrying build of %s on %s...", + build.source_package_name, build.arch_tag) + build.retry() + retry_count += 1 + + if args.priority and can_rescore: + if build.can_be_rescored: + try: + build.rescore(score=args.priority) + except lazr.restfulclient.errors.Unauthorized: + Logger.error( + "You don't have the permissions to rescore builds. Ignoring your rescore request." + ) + can_rescore = False + except lazr.restfulclient.errors.BadRequest: + Logger.info("Cannot rescore build of %s on %s.", + build.source_package_name, build.arch_tag) + + Logger.info("") + if args.retry: + Logger.info("%d package builds retried", retry_count) + sys.exit(0) - # Check permisions (part 1): Rescoring can only be done by buildd admins - can_rescore = args.priority and me.isLpTeamMember("launchpad-buildd-admins") - if args.priority and not can_rescore: - Logger.error( - "You don't have the permissions to rescore builds. Ignoring your rescore request." - ) for pkg in args.packages: try: - pkg = ubuntu_archive.getSourcePackage(pkg, release, pocket) - except PackageNotFoundException as error: - Logger.error(error) + pkg = archive.getPublishedSources( + distro_series=distroseries, + exact_match=True, + pocket=pocket, + source_name=pkg, + status='Published')[0] + except IndexError as error: + Logger.error("No publication found for package %s", pkg) continue # Check permissions (part 2): check upload permissions for the source # package - can_retry = args.retry and me.canUploadPackage( - ubuntu_archive, distroseries, pkg.getPackageName(), pkg.getComponent() + can_retry = args.retry and archive.checkUpload( + component=pkg.component_name, + distroseries=distroseries, + person=launchpad.me, + pocket=pocket, + sourcepackagename=pkg.source_package_name, ) if args.retry and not can_retry: Logger.error( "You don't have the permissions to retry the " "build of '%s'. Ignoring your request.", - pkg.getPackageName(), + pkg.source_package_name, ) Logger.info( "The source version for '%s' in '%s' (%s) is: %s", - pkg.getPackageName(), + pkg.source_package_name, release, pocket, - pkg.getVersion(), + pkg.source_package_version, ) - Logger.info(pkg.getBuildStates(archs)) + Logger.info(getBuildStates(pkg, archs)) if can_retry: - Logger.info(pkg.retryBuilds(archs)) - if args.priority and can_rescore: - Logger.info(pkg.rescoreBuilds(archs, args.priority)) + Logger.info(retryBuilds(pkg, archs)) + if args.priority: + Logger.info(rescoreBuilds(pkg, archs, args.priority)) Logger.info("") if __name__ == "__main__": main() + Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/__init__.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/__init__.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/__init__.cpython-37.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/__init__.cpython-37.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/archive.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/archive.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/config.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/config.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/logger.cpython-37.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/logger.cpython-37.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/misc.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/misc.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/misc.cpython-37.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/misc.cpython-37.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/pullpkg.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/pullpkg.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/rdepends.cpython-37.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/rdepends.cpython-37.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/utils.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/utils.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/__pycache__/version.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/__pycache__/version.cpython-311.pyc differ diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/archive.py ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/archive.py --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/archive.py 2023-07-06 12:00:45.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/archive.py 2023-11-09 09:45:39.000000000 +0000 @@ -946,7 +946,7 @@ class Madison(_WebJSON): urls = { "debian": "https://api.ftp-master.debian.org/madison", - "ubuntu": "http://people.canonical.com/~ubuntu-archive/madison.cgi", + "ubuntu": "https://ubuntu-archive-team.ubuntu.com/madison.cgi", } def __init__(self, distro="debian"): @@ -976,7 +976,7 @@ # Snapshot API # https://anonscm.debian.org/cgit/mirror/snapshot.debian.org.git/plain/API class _Snapshot(_WebJSON): - DEBIAN_COMPONENTS = ["main", "contrib", "non-free"] + DEBIAN_COMPONENTS = ["main", "contrib", "non-free", "non-free-firmware"] def getHostUrl(self): return "http://snapshot.debian.org" Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/__pycache__/__init__.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/__pycache__/__init__.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/__pycache__/__init__.cpython-37.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/__pycache__/__init__.cpython-37.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/__pycache__/lpapicache.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/__pycache__/lpapicache.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/__pycache__/lpapicache.cpython-37.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/__pycache__/lpapicache.cpython-37.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/__pycache__/udtexceptions.cpython-311.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/__pycache__/udtexceptions.cpython-311.pyc differ Binary files /tmp/tmpjs_gbud9/ZmSy5nxjTP/ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/__pycache__/udtexceptions.cpython-37.pyc and /tmp/tmpjs_gbud9/6iuvTeq5qu/ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/__pycache__/udtexceptions.cpython-37.pyc differ diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/lpapicache.py ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/lpapicache.py --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/lp/lpapicache.py 2023-07-05 10:10:26.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/lp/lpapicache.py 2024-03-13 00:01:24.000000000 +0000 @@ -1097,51 +1097,6 @@ for build in builds: self._builds[build.arch_tag] = Build(build) - def getBuildStates(self, archs): - res = [] - - if not self._builds: - self._fetch_builds() - - for arch in archs: - build = self._builds.get(arch) - if build: - res.append(f" {build}") - msg = "\n".join(res) - return f"Build state(s) for '{self.getPackageName()}':\n{msg}" - - def rescoreBuilds(self, archs, score): - res = [] - - if not self._builds: - self._fetch_builds() - - for arch in archs: - build = self._builds.get(arch) - if build: - if build.rescore(score): - res.append(f" {arch}: done") - else: - res.append(f" {arch}: failed") - msg = "\n".join(res) - return f"Rescoring builds of '{self.getPackageName()}' to {score}:\n{msg}" - - def retryBuilds(self, archs): - res = [] - - if not self._builds: - self._fetch_builds() - - for arch in archs: - build = self._builds.get(arch) - if build: - if build.retry(): - res.append(f" {arch}: done") - else: - res.append(f" {arch}: failed") - msg = "\n".join(res) - return f"Retrying builds of '{self.getPackageName()}':\n{msg}" - class BinaryPackagePublishingHistory(BaseWrapper): """ diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/pullpkg.py ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/pullpkg.py --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/pullpkg.py 2023-07-05 10:10:26.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/pullpkg.py 2023-11-09 09:45:39.000000000 +0000 @@ -445,6 +445,37 @@ Logger.info("Found %s", spph.display_name) + # The VCS detection logic was modeled after `apt source` + for key in srcpkg.dsc.keys(): + original_key = key + key = key.lower() + + if key.startswith("vcs-"): + if key == "vcs-browser": + continue + elif key == "vcs-git": + vcs = "Git" + elif key == "vcs-bzr": + vcs = "Bazaar" + else: + continue + + uri = srcpkg.dsc[original_key] + + Logger.warning("\nNOTICE: '%s' packaging is maintained in " + "the '%s' version control system at:\n" + " %s\n" % (package, vcs, uri)) + + if vcs == "Bazaar": + vcscmd = " $ bzr branch " + uri + elif vcs == "Git": + vcscmd = " $ git clone " + uri + + if vcscmd: + Logger.info(f"Please use:\n{vcscmd}\n" + "to retrieve the latest (possibly unreleased) " + "updates to the package.\n") + if pull == PULL_LIST: Logger.info("Source files:") for f in srcpkg.dsc["Files"]: diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/running_autopkgtests.py ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/running_autopkgtests.py --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/running_autopkgtests.py 1970-01-01 00:00:00.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/running_autopkgtests.py 2024-03-12 18:52:03.000000000 +0000 @@ -0,0 +1,91 @@ +# Copyright (C) 2024 Canonical Ltd. +# Author: Chris Peterson +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import datetime +import json +import sys +import urllib +import urllib.request + +URL_RUNNING = "http://autopkgtest.ubuntu.com/static/running.json" +URL_QUEUED = "http://autopkgtest.ubuntu.com/queues.json" + + +def _get_jobs(url: str) -> dict: + request = urllib.request.Request( + url, + headers={"Cache-Control": "max-age-0"}, + ) + with urllib.request.urlopen(request) as response: + data = response.read() + jobs = json.loads(data.decode("utf-8")) + + return jobs + + +def get_running(): + jobs = _get_jobs(URL_RUNNING) + + running = [] + for pkg in jobs: + for handle in jobs[pkg]: + for series in jobs[pkg][handle]: + for arch in jobs[pkg][handle][series]: + jobinfo = jobs[pkg][handle][series][arch] + triggers = ",".join(jobinfo[0].get("triggers", "-")) + ppas = ",".join(jobinfo[0].get("ppas", "-")) + time = jobinfo[1] + env = jobinfo[0].get("env", "-") + time = str(datetime.timedelta(seconds=jobinfo[1])) + try: + line = f"R {time:6} {pkg:30} {'-':10} {series:8} {arch:8} {ppas:31} {triggers} {env}\n" + running.append((jobinfo[1], line)) + except BrokenPipeError: + sys.exit(1) + + output = "" + for time, row in sorted(running, reverse=True): + output += f"{row}" + + return output + + +def get_queued(): + queues = _get_jobs(URL_QUEUED) + output = "" + for origin in queues: + for series in queues[origin]: + for arch in queues[origin][series]: + n = 0 + for key in queues[origin][series][arch]: + if key == "private job": + pkg = triggers = ppas = "private job" + else: + (pkg, json_data) = key.split(maxsplit=1) + try: + jobinfo = json.loads(json_data) + triggers = ",".join(jobinfo.get("triggers", "-")) + ppas = ",".join(jobinfo.get("ppas", "-")) + except json.decoder.JSONDecodeError: + pkg = triggers = ppas = "failed to parse" + continue + + n = n + 1 + try: + output += f"Q{n:04d} {'-:--':>6} {pkg:30} {origin:10} {series:8} {arch:8} {ppas:31} {triggers}\n" + except BrokenPipeError: + sys.exit(1) + return output diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/test/test_running_autopkgtests.py ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/test/test_running_autopkgtests.py --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/test/test_running_autopkgtests.py 1970-01-01 00:00:00.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/test/test_running_autopkgtests.py 2024-03-12 18:52:03.000000000 +0000 @@ -0,0 +1,109 @@ +# Copyright (C) 2024 Canonical Ltd. +# Author: Chris Peterson +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +""" Tests for running_autopkgtests +Tests using cached data from autopkgtest servers. + +These tests only ensure code changes don't change parsing behavior +of the response data. If the response format changes, then the cached +responses will need to change as well. +""" + +import unittest +from unittest.mock import patch + +from ubuntutools.running_autopkgtests import ( + URL_QUEUED, + URL_RUNNING, + _get_jobs, + get_queued, + get_running, +) + +# Cached binary response data from autopkgtest server +RUN_DATA = b'{"pyatem": { "submit-time_2024-01-19 19:37:36;triggers_[\'python3-defaults/3.12.1-0ubuntu1\'];": {"noble": {"arm64": [{"triggers": ["python3-defaults/3.12.1-0ubuntu1"], "submit-time": "2024-01-19 19:37:36"}, 380, ""]}}}}' +QUEUED_DATA = b'{"ubuntu": {"noble": {"arm64": ["libobject-accessor-perl {\\"requester\\": \\"someone\\", \\"submit-time\\": \\"2024-01-18 01:08:55\\", \\"triggers\\": [\\"perl/5.38.2-3\\", \\"liblocale-gettext-perl/1.07-6build1\\"]}"]}}}' + +# Expected result(s) of parsing the above JSON data +RUNNING_JOB = { + "pyatem": { + "submit-time_2024-01-19 19:37:36;triggers_['python3-defaults/3.12.1-0ubuntu1'];": { + "noble": { + "arm64": [ + { + "triggers": ["python3-defaults/3.12.1-0ubuntu1"], + "submit-time": "2024-01-19 19:37:36", + }, + 380, + "", + ] + } + } + } +} + +QUEUED_JOB = { + "ubuntu": { + "noble": { + "arm64": [ + 'libobject-accessor-perl {"requester": "someone", "submit-time": "2024-01-18 01:08:55", "triggers": ["perl/5.38.2-3", "liblocale-gettext-perl/1.07-6build1"]}', + ] + } + } +} + + +PRIVATE_JOB = {"ppa": {"noble": {"arm64": ["private job"]}}} + + +# Expected textual output of the program based on the above data +RUNNING_OUTPUT = "R 0:06:20 pyatem - noble arm64 - python3-defaults/3.12.1-0ubuntu1 -\n" +QUEUED_OUTPUT = "Q0001 -:-- libobject-accessor-perl ubuntu noble arm64 - perl/5.38.2-3,liblocale-gettext-perl/1.07-6build1\n" +PRIVATE_OUTPUT = "Q0001 -:-- private job ppa noble arm64 private job private job\n" + + +class RunningAutopkgtestTestCase(unittest.TestCase): + """Assert helper functions parse data correctly""" + + maxDiff = None + + @patch("urllib.request.urlopen") + def test_get_running_jobs(self, mock_response): + """Test: Correctly parse autopkgtest json data for running tests""" + mock_response.return_value.__enter__.return_value.read.return_value = RUN_DATA + jobs = _get_jobs(URL_RUNNING) + self.assertEqual(RUNNING_JOB, jobs) + + @patch("urllib.request.urlopen") + def test_get_queued_jobs(self, mock_response): + """Test: Correctly parse autopkgtest json data for queued tests""" + mock_response.return_value.__enter__.return_value.read.return_value = QUEUED_DATA + jobs = _get_jobs(URL_QUEUED) + self.assertEqual(QUEUED_JOB, jobs) + + def test_get_running_output(self): + """Test: Correctly print running tests""" + with patch("ubuntutools.running_autopkgtests._get_jobs", return_value=RUNNING_JOB): + self.assertEqual(get_running(), RUNNING_OUTPUT) + + def test_get_queued_output(self): + """Test: Correctly print queued tests""" + with patch("ubuntutools.running_autopkgtests._get_jobs", return_value=QUEUED_JOB): + self.assertEqual(get_queued(), QUEUED_OUTPUT) + + def test_private_queued_job(self): + """Test: Correctly print queued private job""" + with patch("ubuntutools.running_autopkgtests._get_jobs", return_value=PRIVATE_JOB): + self.assertEqual(get_queued(), PRIVATE_OUTPUT) diff -Nru ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/utils.py ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/utils.py --- ubuntu-dev-tools-0.193ubuntu4~22.04.2/ubuntutools/utils.py 1970-01-01 00:00:00.000000000 +0000 +++ ubuntu-dev-tools-0.201ubuntu2~22.04.1/ubuntutools/utils.py 2024-01-29 16:33:27.000000000 +0000 @@ -0,0 +1,82 @@ +# Copyright (C) 2019-2023 Canonical Ltd. +# Author: Brian Murray et al. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Portions of archive related code that is re-used by various tools.""" + +from datetime import datetime +import os +import re +import urllib.request + +import dateutil.parser +from dateutil.tz import tzutc + + +def get_cache_dir(): + cache_dir = os.environ.get('XDG_CACHE_HOME', + os.path.expanduser(os.path.join('~', '.cache'))) + uat_cache = os.path.join(cache_dir, 'ubuntu-archive-tools') + os.makedirs(uat_cache, exist_ok=True) + return uat_cache + + +def get_url(url, force_cached): + ''' Return file to the URL, possibly caching it + ''' + cache_file = None + + # ignore bileto urls wrt caching, they're usually too small to matter + # and we don't do proper cache expiry + m = re.search('ubuntu-archive-team.ubuntu.com/proposed-migration/' + '([^/]*)/([^/]*)', + url) + if m: + cache_dir = get_cache_dir() + cache_file = os.path.join(cache_dir, '%s_%s' % (m.group(1), m.group(2))) + else: + # test logs can be cached, too + m = re.search( + 'https://autopkgtest.ubuntu.com/results/autopkgtest-[^/]*/([^/]*)/([^/]*)' + '/[a-z0-9]*/([^/]*)/([_a-f0-9]*)@/log.gz', + url) + if m: + cache_dir = get_cache_dir() + cache_file = os.path.join( + cache_dir, '%s_%s_%s_%s.gz' % ( + m.group(1), m.group(2), m.group(3), m.group(4))) + + if cache_file: + try: + prev_mtime = os.stat(cache_file).st_mtime + except FileNotFoundError: + prev_mtime = 0 + prev_timestamp = datetime.fromtimestamp(prev_mtime, tz=tzutc()) + new_timestamp = datetime.now(tz=tzutc()).timestamp() + if force_cached: + return open(cache_file, 'rb') + + f = urllib.request.urlopen(url) + + if cache_file: + remote_ts = dateutil.parser.parse(f.headers['last-modified']) + if remote_ts > prev_timestamp: + with open('%s.new' % cache_file, 'wb') as new_cache: + for line in f: + new_cache.write(line) + os.rename('%s.new' % cache_file, cache_file) + os.utime(cache_file, times=(new_timestamp, new_timestamp)) + f.close() + f = open(cache_file, 'rb') + return f