diff -Nru piuparts-1.1.1build1/conf/crontab-slave.in piuparts-1.1.2/conf/crontab-slave.in --- piuparts-1.1.1build1/conf/crontab-slave.in 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/conf/crontab-slave.in 2021-01-19 15:17:16.000000000 +0000 @@ -6,7 +6,8 @@ # # start slave_run automatically after reboots # -#@reboot slave_cleanup ; slave_run +@reboot slave_cleanup ; slave_run + #42 * * * * slave_cleanup ; ! slave_run >/dev/null 2>&1 || echo "piuparts-slave started by cron" # diff -Nru piuparts-1.1.1build1/debian/changelog piuparts-1.1.2/debian/changelog --- piuparts-1.1.1build1/debian/changelog 2020-02-08 11:17:06.000000000 +0000 +++ piuparts-1.1.2/debian/changelog 2021-01-19 15:17:25.000000000 +0000 @@ -1,8 +1,45 @@ -piuparts (1.1.1build1) focal; urgency=medium +piuparts (1.1.2) unstable; urgency=medium - * No-change rebuild with fixed binutils on arm64. + * Team upload. + + [ Nis Martensen ] + * piuparts.py: + - prevent KeyError in diff_meta_data. (Closes: #968842) + - do not crash on non-utf-8 command output. + * piuparts-analyze: unbreak sorting of version lists. + * autopkgtest: add a must-fail test. + * d/p-[ms].lintian-overrides: update to python3/ + + [ Nicolas Dandrimont ] + * Add exemption for /var/lib/dpkg/cmethopt. (Closes: #953756) + * piuparts-analyze: + - print full exceptions when failing. + - fetch all bugs in bulk instead of package by package. + - only wait for 60 seconds when a given section is busy. + * Add myself to uploaders. + * d/rules: Add workaround for dh-dwz bug #933541. + * Set Description last when generating a dummy package. + * Start piuparts-slave automatically after reboots again. + * Add piu-slave-conova-01. + * Add news entry for piu-slave-conova-01. + * Move piu-slave-ubc-01 to deb.d.o. + + [ Ansgar ] + * piuparts-report.py: fix link to states.png. + + [ Stuart Prescott ] + * Add timeout when fetching packages files. + + [ Julien Cristau ] + * Make the summary generation atomic. + + [ Holger Levsen ] + * Bump standards version to 4.5.1, no changes needed. + + [ Debian Janitor ] + * Fix day-of-week for changelog entry 0.16-1. - -- Matthias Klose Sat, 08 Feb 2020 11:17:06 +0000 + -- Holger Levsen Tue, 19 Jan 2021 16:17:25 +0100 piuparts (1.1.1) unstable; urgency=medium @@ -3404,7 +3441,7 @@ * debian/control: Made the dependency on debootstrap be versioned. Closes: #355875. - -- Lars Wirzenius Fri, 22 Mar 2006 22:40:00 +0200 + -- Lars Wirzenius Wed, 22 Mar 2006 22:40:00 +0200 piuparts (0.15-1) unstable; urgency=low diff -Nru piuparts-1.1.1build1/debian/control piuparts-1.1.2/debian/control --- piuparts-1.1.1build1/debian/control 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/debian/control 2021-01-19 15:17:16.000000000 +0000 @@ -1,8 +1,9 @@ Source: piuparts Maintainer: piuparts developers team +Uploaders: Nicolas Dandrimont Section: devel Priority: optional -Standards-Version: 4.4.1 +Standards-Version: 4.5.1 Rules-Requires-Root: no Build-Depends: debhelper-compat (= 12), diff -Nru piuparts-1.1.1build1/debian/piuparts-master.lintian-overrides piuparts-1.1.2/debian/piuparts-master.lintian-overrides --- piuparts-1.1.1build1/debian/piuparts-master.lintian-overrides 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/debian/piuparts-master.lintian-overrides 2021-01-19 15:17:16.000000000 +0000 @@ -1,5 +1,5 @@ # It's ok, because it depends on piuparts-common that has the dependency. -piuparts-master: python-script-but-no-python-dep usr/share/piuparts/master/detect_well_known_errors #!/usr/bin/python3 -piuparts-master: python-script-but-no-python-dep usr/share/piuparts/piuparts-analyze #!/usr/bin/python3 -piuparts-master: python-script-but-no-python-dep usr/share/piuparts/piuparts-master-backend #!/usr/bin/python3 -piuparts-master: python-script-but-no-python-dep usr/share/piuparts/piuparts-report #!/usr/bin/python3 +piuparts-master: python3-script-but-no-python3-dep usr/share/piuparts/master/detect_well_known_errors #!/usr/bin/python3 +piuparts-master: python3-script-but-no-python3-dep usr/share/piuparts/piuparts-analyze #!/usr/bin/python3 +piuparts-master: python3-script-but-no-python3-dep usr/share/piuparts/piuparts-master-backend #!/usr/bin/python3 +piuparts-master: python3-script-but-no-python3-dep usr/share/piuparts/piuparts-report #!/usr/bin/python3 diff -Nru piuparts-1.1.1build1/debian/piuparts-slave.lintian-overrides piuparts-1.1.2/debian/piuparts-slave.lintian-overrides --- piuparts-1.1.1build1/debian/piuparts-slave.lintian-overrides 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/debian/piuparts-slave.lintian-overrides 2021-01-19 15:17:16.000000000 +0000 @@ -1,2 +1,2 @@ # It's ok, because it depends on piuparts-common that has the dependency. -piuparts-slave: python-script-but-no-python-dep usr/share/piuparts/piuparts-slave #!/usr/bin/python3 +piuparts-slave: python3-script-but-no-python3-dep usr/share/piuparts/piuparts-slave #!/usr/bin/python3 diff -Nru piuparts-1.1.1build1/debian/rules piuparts-1.1.2/debian/rules --- piuparts-1.1.1build1/debian/rules 2020-01-14 12:47:48.000000000 +0000 +++ piuparts-1.1.2/debian/rules 2021-01-19 15:17:16.000000000 +0000 @@ -17,3 +17,8 @@ override_dh_python3: dh_python3 -p piuparts-master -p piuparts-slave /usr/share/piuparts --shebang=/usr/bin/python3 dh_python3 -N piuparts-master -N piuparts-slave + +override_dh_dwz: + # Workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=933541 + # (this workaround can be dropped with debhelper from buster-backports) + : diff -Nru piuparts-1.1.1build1/debian/tests/smoke-test piuparts-1.1.2/debian/tests/smoke-test --- piuparts-1.1.1build1/debian/tests/smoke-test 2020-01-14 12:47:48.000000000 +0000 +++ piuparts-1.1.2/debian/tests/smoke-test 2021-01-19 15:17:16.000000000 +0000 @@ -16,6 +16,9 @@ trap "rm -rf $WORKDIR" 0 INT QUIT ABRT PIPE TERM cd $WORKDIR + +# set up a very simple test package + mkdir -p t/DEBIAN t/usr t/etc cat >t/DEBIAN/control <f/DEBIAN/control < +Priority: optional +Architecture: all +Installed-Size: 0 +Description: Auto Package Test Fail Dummy + Extremely simple binary package for piuparts testing - fail version +EOF + +cat >f/DEBIAN/postinst < - Since 2005 piuparts is run systematically on all packages in Debian, at first only on unstable. Today piuparts.debian.org is testing many suites by running piuparts in master/slave mode. This setup currently consists of two hosts: pejacevic.debian.org and and piu-slave-ubc-01.debian.org: + Since 2005 piuparts is run systematically on all packages in Debian, at first only on unstable. Today piuparts.debian.org is testing many suites by running piuparts in master/slave mode. This setup currently consists of three hosts: pejacevic.debian.org, piu-slave-ubc-01.debian.org and piu-slave-conova-01.debian.org:
    -
  • pejacevic acts as the piuparts-master, which is responsible for scheduling test jobs to the slaves. The other main task is to generate the reports which are served via https://piuparts.debian.org.
    • -
  • piu-slave-ubc-01 runs four piuparts-slave instances, which then run piuparts itself.
    • +
  • pejacevic acts as the piuparts-master, which is responsible for scheduling test jobs to the slaves. The other main task is to generate the reports which are served via https://piuparts.debian.org;
    • +
  • piu-slave-ubc-01 runs four piuparts-slave instances, which then run piuparts itself;
    • +
  • piu-slave-conova-01 also runs four piuparts-slave instances, which then run piuparts itself.
diff -Nru piuparts-1.1.1build1/htdocs/news.tpl piuparts-1.1.2/htdocs/news.tpl --- piuparts-1.1.1build1/htdocs/news.tpl 2020-01-08 19:55:59.000000000 +0000 +++ piuparts-1.1.2/htdocs/news.tpl 2021-01-19 15:17:16.000000000 +0000 @@ -6,6 +6,12 @@ + 2020-04-29 + Thanks to DSA, a new worker host, piu-slave-conova-01, has been introduced. It will run 4 parallel worker processes on the amd64 arch. + + + + 2020-01-08 After maintaining piuparts.debian.org for more than a decade, Holger Levsen resigns from doing so and also steps back from maintaining the piuparts source package in Debian, after doing this for even more years. It was really a lot of fun, I learned a lot and am thankful for being able to contribute this service to improve Debian. Many thanks to all who contributed to it, especially Andreas Beckmann. And also many thanks to those, who will keep it running in the future! diff -Nru piuparts-1.1.1build1/instances/Makefile piuparts-1.1.2/instances/Makefile --- piuparts-1.1.1build1/instances/Makefile 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/instances/Makefile 2021-01-19 15:17:16.000000000 +0000 @@ -3,11 +3,11 @@ ARCH_piu-slave-1und1-01 = i386 MIRROR_piu-slave-1und1-01 = http://mirror.1und1.de/debian/ -MIRROR_piu-slave-ubc-01 = http://mirror-ubc.debian.org/debian/ GENCONF = piuparts.conf.pejacevic GENCONF += piuparts.conf.piu-slave-1und1-01 GENCONF += piuparts.conf.piu-slave-ubc-01 +GENCONF += piuparts.conf.piu-slave-conova-01 all: $(GENCONF) diff -Nru piuparts-1.1.1build1/instances/piuparts.conf-header.piu-slave-conova-01 piuparts-1.1.2/instances/piuparts.conf-header.piu-slave-conova-01 --- piuparts-1.1.1build1/instances/piuparts.conf-header.piu-slave-conova-01 1970-01-01 00:00:00.000000000 +0000 +++ piuparts-1.1.2/instances/piuparts.conf-header.piu-slave-conova-01 2021-01-19 15:17:16.000000000 +0000 @@ -0,0 +1,8 @@ +# +# This is the configuration file for piuparts running in master-slave mode on +# - piu-slave-conova-01.debian.org (as slave). +# Except for the mirror configuration this file is identical to the one +# for pejacevic and piu-slave-ubc-01. +# +# For more information on this setup see https://salsa.debian.org/debian/piuparts/blob/develop/README_pejacevic.txt +# diff -Nru piuparts-1.1.1build1/piuparts-analyze.py piuparts-1.1.2/piuparts-analyze.py --- piuparts-1.1.1build1/piuparts-analyze.py 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/piuparts-analyze.py 2021-01-19 15:17:16.000000000 +0000 @@ -4,6 +4,7 @@ # Copyright 2005 Lars Wirzenius (liw@iki.fi) # Copyright 2011 Mika Pflüger (debian@mikapflueger.de) # Copyright © 2012-2017 Andreas Beckmann (anbe@debian.org) +# Copyright © 2020 Nicolas Dandrimont (nicolas@dandrimont.eu) # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -31,18 +32,19 @@ """ from __future__ import print_function +from collections import defaultdict +import fcntl import os import sys import time import re import shutil -import subprocess -import fcntl +import traceback import debianbts import apt_pkg -from signal import alarm, signal, SIGALRM from collections import deque +from functools import cmp_to_key import piupartslib.conf from piupartslib.conf import MissingSection @@ -57,78 +59,71 @@ ############################################################################ -class BTS_Timeout(Exception): - pass - - -def alarm_handler(signum, frame): - raise BTS_Timeout - - class PiupartsBTS(): def __init__(self): - self._bugs_usertagged_piuparts = None - self._bugs_in_package = dict() - self._bugs_affecting_package = dict() - self._bug_versions = dict() + self._fetched = False + self._bugs = {} + self._bugs_in_package = defaultdict(set) + self._bugs_affecting_package = defaultdict(set) + self._bug_versions = {} self._queries = 0 - self._misses = 0 - def all_piuparts_bugs(self): - if self._bugs_usertagged_piuparts is None: - self._bugs_usertagged_piuparts = debianbts.get_usertag("debian-qa@lists.debian.org", ['piuparts'])['piuparts'] - return self._bugs_usertagged_piuparts + def fetch_bug_data(self): + if self._fetched: + return + self._fetched = True + bug_nums = debianbts.get_usertag("debian-qa@lists.debian.org", ['piuparts'])['piuparts'] + bugs = debianbts.get_status(bug_nums) + for bug in bugs: + bug_num = bug.bug_num + + self._bugs[bug_num] = bug + + # Populate bug number -> versions map + versions = [] + for version in bug.found_versions: + # debianbts returns found versions in the format package/1.2.3 or 1.2.3 which will become 1.2.3 + v = version.rsplit('/', 1)[-1] + if v == 'None': + continue + versions.append(v) + if versions: + versions.sort(key=cmp_to_key(apt_pkg.version_compare)) + self._bug_versions[bug_num] = versions + + # Populate package -> bug map + for package in bug.package.split(','): + if package.startswith('src:'): + package = package[4:] + self._bugs_in_package[package].add(bug_num) + + # Populate affected package -> bug map + for package in bug.affects: + if package.startswith('src:'): + package = package[4:] + self._bugs_affecting_package[package].add(bug_num) def bugs_in(self, package): - if not package in self._bugs_in_package: - self._misses += 1 - signal(SIGALRM, alarm_handler) - alarm(120) - bugs = debianbts.get_bugs(package=package, bugs=self.all_piuparts_bugs(), archive='both') - bugs += debianbts.get_bugs(package='src:' + package, bugs=self.all_piuparts_bugs(), archive='both') - alarm(0) - self._bugs_in_package[package] = sorted(set(bugs), reverse=True) self._queries += 1 + self.fetch_bug_data() return self._bugs_in_package[package] def bugs_affecting(self, package): - if not package in self._bugs_affecting_package: - self._misses += 1 - signal(SIGALRM, alarm_handler) - alarm(120) - bugs = debianbts.get_bugs(affects=package, bugs=self.all_piuparts_bugs(), archive='both') - bugs += debianbts.get_bugs(affects='src:' + package, bugs=self.all_piuparts_bugs(), archive='both') - alarm(0) - self._bugs_affecting_package[package] = sorted(set(bugs), reverse=True) self._queries += 1 + self.fetch_bug_data() return self._bugs_affecting_package[package] def bug_versions(self, bug): """Gets a list of only the version numbers for which the bug is found. Newest versions are returned first.""" - # debianbts returns it in the format package/1.2.3 or 1.2.3 which will become 1.2.3 - if not bug in self._bug_versions: - self._misses += 1 - signal(SIGALRM, alarm_handler) - alarm(60) - found_versions = debianbts.get_status([bug])[0].found_versions - alarm(0) - versions = [] - for found_version in found_versions: - v = found_version.rsplit('/', 1)[-1] - if v == "None": - # ignore $DISTRO/None versions - pass - else: - versions.append(v) - self._bug_versions[bug] = list(reversed(sorted(versions, cmp=apt_pkg.version_compare))) or ['~'] self._queries += 1 - return self._bug_versions[bug] + self.fetch_bug_data() + return self._bug_versions.get(bug, ['~']) def print_stats(self): - print("PiupartsBTS: %d queries, %d forwarded to debianbts" % (self._queries, self._misses)) + print("PiupartsBTS: %d queries" % self._queries) piupartsbts = PiupartsBTS() @@ -271,7 +266,8 @@ moved = False abugs = piupartsbts.bugs_affecting(pname) bugs = piupartsbts.bugs_in(pname) - for bug in abugs + bugs: + all_bugs = sorted(abugs | bugs, reverse=True) + for bug in all_bugs: if moved: break if bug in abugs: @@ -306,13 +302,12 @@ bts_update_found(bug, pversion) break if not moved: - write_bug_file(failed_log, abugs + bugs) + write_bug_file(failed_log, all_bugs) except KeyboardInterrupt: raise - except: + except Exception: print('ERROR processing %s' % failed_log) - print(sys.exc_info()[0]) - alarm(0) + traceback.print_exc() def main(): @@ -359,7 +354,7 @@ os.chdir(oldcwd) except Busy: print("Section is busy") - todo.append((section_name, time.time() + 300)) + todo.append((section_name, time.time() + 60)) except MissingSection as e: print("Configuration Error in section '%s': %s" % (section_name, e)) print("") diff -Nru piuparts-1.1.1build1/piupartslib/__init__.py piuparts-1.1.2/piupartslib/__init__.py --- piuparts-1.1.1build1/piupartslib/__init__.py 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/piupartslib/__init__.py 2021-01-19 15:17:16.000000000 +0000 @@ -93,7 +93,7 @@ error = None for ext in ['.xz', '.bz2', '.gz', '']: try: - socket = urllib.request.urlopen(url + ext) + socket = urllib.request.urlopen(url + ext, timeout=30) except urllib.error.HTTPError as e: error = e else: diff -Nru piuparts-1.1.1build1/piupartslib/pkgsummary.py piuparts-1.1.2/piupartslib/pkgsummary.py --- piuparts-1.1.1build1/piupartslib/pkgsummary.py 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/piupartslib/pkgsummary.py 2021-01-19 15:17:16.000000000 +0000 @@ -83,6 +83,7 @@ import json import datetime from collections import namedtuple, defaultdict +import os import six @@ -221,8 +222,9 @@ def write_summary(summary, fname): - with open(fname, 'w') as fl: + with open(fname + '.tmp', 'w') as fl: json.dump(summary, fl, sort_keys=True, indent=1) + os.rename(fname + '.tmp', fname) def read_summary(fname): diff -Nru piuparts-1.1.1build1/piuparts.py piuparts-1.1.2/piuparts.py --- piuparts-1.1.1build1/piuparts.py 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/piuparts.py 2021-01-19 15:17:16.000000000 +0000 @@ -297,6 +297,7 @@ "/var/lib/dpkg/arch", "/var/lib/dpkg/available", "/var/lib/dpkg/available-old", + "/var/lib/dpkg/cmethopt", # Bug#953756 "/var/lib/dpkg/diversions", "/var/lib/dpkg/diversions-old", "/var/lib/dpkg/lock", @@ -558,7 +559,7 @@ devnull = open('/dev/null', 'r') p = subprocess.Popen(command, env=env, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True) + universal_newlines=True, errors="backslashreplace") output = "" excessive_output = False if timeout > 0: @@ -655,15 +656,16 @@ control['Architecture'] = arch control['Maintainer'] = ('piuparts developers team ' '') - control['Description'] = ('Dummy package to satisfy dependencies - ' - 'created by piuparts\n' - ' This package was created automatically by ' - 'piuparts and can safely be removed') if depends: control['Depends'] = depends if conflicts: control['Conflicts'] = conflicts + control['Description'] = ('Dummy package to satisfy dependencies - ' + 'created by piuparts\n' + ' This package was created automatically by ' + 'piuparts and can safely be removed') + create_file(os.path.join(tmpdir, name, 'DEBIAN', 'control'), control.dump()) @@ -2011,11 +2013,11 @@ name = name[1:] else: verbose = False - if name in tree1: + if name in tree1_c: if verbose: logging.info("IGNORED PATH@1: %s" % name) del tree1_c[name] - if name in tree2: + if name in tree2_c: if verbose: logging.info("IGNORED PATH@2: %s" % name) del tree2_c[name] @@ -2029,20 +2031,20 @@ pat = re.compile(pattern) for name in tree1.keys(): m = pat.search(name) - if m: + if m and name in tree1_c: if verbose: logging.info("IGNORED PATH@1: %s" % name) del tree1_c[name] for name in tree2.keys(): m = pat.search(name) - if m: + if m and name in tree2_c: if verbose: logging.info("IGNORED PATH@2: %s" % name) del tree2_c[name] modified = [] for name in tree1.keys(): - if name in tree2_c: + if name in tree1_c and name in tree2_c: if objects_are_different(tree1[name], tree2[name]): if not quiet: logging.debug("Modified(user, group, mode, size, target): %s expected%s != found%s" % diff -Nru piuparts-1.1.1build1/piuparts-report.py piuparts-1.1.2/piuparts-report.py --- piuparts-1.1.1build1/piuparts-report.py 2020-01-08 19:31:05.000000000 +0000 +++ piuparts-1.1.2/piuparts-report.py 2021-01-19 15:17:16.000000000 +0000 @@ -761,7 +761,7 @@ % (doc_root, current_section) tablerows += "by source package\n" \ % (doc_root, current_section) - tablerows += "states graph\n" \ + tablerows += "states graph\n" \ % (doc_root, current_section) tablerows += "all tested suites" for section in section_names: