diff -Nru autopkgtest-5.15/debian/changelog autopkgtest-5.16/debian/changelog --- autopkgtest-5.15/debian/changelog 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/debian/changelog 2021-01-24 20:15:20.000000000 +0000 @@ -1,3 +1,62 @@ +autopkgtest (5.16) unstable; urgency=medium + + [ Paul Gevers ] + * Fix reported arch in Unsupported message + + [ Simon McVittie ] + * tests/ssh-setup-lxd: Fix a typo + * shell scripts: Quote more defensively + * tests/pycodestyle: Prefer command -v instead of which + * shell scripts: Prefer $() instead of backticks + * setup-testbed: Avoid test -a + * setup-testbed: Silence some shellcheck false-positives + * setup-testbed: Be more careful with printf + * setup-testbed: Ensure that $proxy is always initialized + * setup-testbed: Prefer POSIX grep -E over GNU-specific egrep + * setup-testbed: Quote trap command differently + * tests: Add shellcheck test + * tests: Silence some mypy warnings + * tests: Add test that runs mypy across the codebase + * d/copyright: Update + * Use shlex.quote in preference to undocumented pipes.quote + + [ Ryutaroh Matsumoto ] + * autopkgtest-build-qemu: Install a PAE kernel on i386 (Closes: #973662) + * autopkgtest-build-qemu: Install LPAE kernel on armhf (Closes: #973592) + + [ Simon McVittie ] + * lib: Factor out qemu virtual machine setup + * autopkgtest_qemu: Add QemuImage objects + * buildvm-ubuntu-cloud: Use Qemu object + * autopkgtest-build-qemu: Rewrite in Python + * autopkgtest-buildvm-ubuntu-cloud: Make return of None explicit + * autopkgtest-buildvm-ubuntu-cloud: Fail if release cannot be determined + * autopkgtest-buildvm-ubuntu-cloud: Replace which(1) with shutil.which + * autopkgtest_qemu: Set workdir before preparing overlay + + [ Ryutaroh Matsumoto ] + * qemu: Forward stdout, stderr to stderr of parent process + * qemu: Extend timeout for checking python availability + * qemu: Enable discard operation on virtio disk devices + + [ Simon McVittie ] + * qemu: Allow passing `uname -m` output to get_default_qemu_command + * tests: Add a unit test for qemu library code + * qemu: Select qemu-system-arm for ARM CPUs + + [ Graham Inggs ] + * autpkgtest_args: autoremove after dist-upgrade + + [ Balint Reczey ] + * Handle exception thrown by create_testinfo() (Closes: #927146) + (LP: #1907803) + + [ Sébastien Delafond ] + * virt-lxc: attempt to cleanly shutdown the container first, and only + then pass --force to lxc-destroy (Closes: #979238) + + -- Paul Gevers Sun, 24 Jan 2021 21:15:20 +0100 + autopkgtest (5.15) unstable; urgency=medium [ Sebastien Delafond ] diff -Nru autopkgtest-5.15/debian/control autopkgtest-5.16/debian/control --- autopkgtest-5.15/debian/control 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/debian/control 2021-01-24 20:15:20.000000000 +0000 @@ -11,7 +11,7 @@ procps, pycodestyle | pep8, pyflakes3, - python3 (>= 3.1), + python3 (>= 3.3), python3-debian, python3-docutils, python3-mock diff -Nru autopkgtest-5.15/debian/copyright autopkgtest-5.16/debian/copyright --- autopkgtest-5.15/debian/copyright 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/debian/copyright 2021-01-24 20:15:20.000000000 +0000 @@ -1,7 +1,17 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Files: * -Copyright: Copyright (C) 2006-2014 Canonical Ltd. +Copyright: + 2006-2018 Canonical Ltd. and others + 2012-2017 Martin Pitt + 2016-2020 Simon McVittie + 2016-2020 Antonio Terceiro + 2017 Jiri Palecek + 2017-2020 Collabora Ltd. + 2018 Thadeu Lima de Souza Cascardo + 2019 Michael Biebl + 2019 Raphaël Hertzog + 2019 Sébastien Delafond License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff -Nru autopkgtest-5.15/debian/rules autopkgtest-5.16/debian/rules --- autopkgtest-5.15/debian/rules 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/debian/rules 2021-01-24 20:15:20.000000000 +0000 @@ -36,7 +36,9 @@ override_dh_auto_test: ifeq (, $(findstring nocheck, $(DEB_BUILD_OPTIONS))) if type pyflakes3 >/dev/null 2>&1; then tests/pyflakes; else echo "pyflakes3 not available, skipping"; fi + tests/mypy tests/pycodestyle || true + tests/shellcheck tests/testdesc tests/autopkgtest_args env NO_PKG_MANGLE=1 tests/autopkgtest NullRunner diff -Nru autopkgtest-5.15/lib/adt_testbed.py autopkgtest-5.16/lib/adt_testbed.py --- autopkgtest-5.15/lib/adt_testbed.py 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/lib/adt_testbed.py 2021-01-24 20:15:20.000000000 +0000 @@ -24,9 +24,9 @@ import sys import errno import time -import pipes import traceback import re +import shlex import signal import subprocess import tempfile @@ -113,7 +113,7 @@ # log command line invocation for the log adtlog.info('host %s; command line: %s' % ( - os.uname()[1], ' '.join([pipes.quote(w) for w in sys.argv]))) + os.uname()[1], ' '.join([shlex.quote(w) for w in sys.argv]))) self.sp = subprocess.Popen(self.vserver_argv, stdin=subprocess.PIPE, diff -Nru autopkgtest-5.15/lib/autopkgtest_args.py autopkgtest-5.16/lib/autopkgtest_args.py --- autopkgtest-5.15/lib/autopkgtest_args.py 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/lib/autopkgtest_args.py 2021-01-24 20:15:20.000000000 +0000 @@ -254,7 +254,8 @@ '{ [ "${O%404*Not Found*}" = "$O" ] || exit 100; sleep 15; apt-get update; }''' ' || { sleep 60; apt-get update; } || false)' ' && $(which eatmydata || true) apt-get dist-upgrade -y -o ' - 'Dpkg::Options::="--force-confnew"', + 'Dpkg::Options::="--force-confnew"' + ' && $(which eatmydata || true) apt-get --purge autoremove -y', help='Run apt update/dist-upgrade before the tests') g_setup.add_argument('--setup-commands-boot', metavar='COMMANDS_OR_PATH', action='append', default=[], diff -Nru autopkgtest-5.15/lib/autopkgtest_qemu.py autopkgtest-5.16/lib/autopkgtest_qemu.py --- autopkgtest-5.15/lib/autopkgtest_qemu.py 1970-01-01 00:00:00.000000000 +0000 +++ autopkgtest-5.16/lib/autopkgtest_qemu.py 2021-01-24 20:15:20.000000000 +0000 @@ -0,0 +1,385 @@ +#!/usr/bin/python3 +# +# This is not a stable API; for use within autopkgtest only. +# +# Part of autopkgtest. +# autopkgtest is a tool for testing Debian binary packages. +# +# Copyright 2006-2016 Canonical Ltd. +# Copyright 2016-2020 Simon McVittie +# Copyright 2017 Martin Pitt +# Copyright 2017 Jiri Palecek +# Copyright 2017-2018 Collabora Ltd. +# Copyright 2018 Thadeu Lima de Souza Cascardo +# Copyright 2019 Michael Biebl +# Copyright 2019 Raphaël Hertzog +# +# autopkgtest-virt-qemu was developed by +# Martin Pitt +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# See the file CREDITS for a full list of credits information (often +# installed as /usr/share/doc/autopkgtest/CREDITS). + +import errno +import fcntl +import json +import os +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import time +from typing import ( + List, + Optional, + Sequence, + Union, +) + +import VirtSubproc +import adtlog + + +def find_free_port(start: int) -> int: + '''Find an unused port in the range [start, start+50)''' + + for p in range(start, start + 50): + adtlog.debug('find_free_port: trying %i' % p) + try: + lockfile = '/tmp/autopkgtest-virt-qemu.port.%i' % p + f = None + try: + f = open(lockfile, 'x') + os.unlink(lockfile) + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + except (IOError, OSError): + adtlog.debug('find_free_port: %i is locked' % p) + continue + finally: + if f: + f.close() + + s = socket.create_connection(('127.0.0.1', p)) + # if that works, the port is taken + s.close() + continue + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + adtlog.debug('find_free_port: %i is free' % p) + return p + else: + pass + + adtlog.debug('find_free_port: all ports are taken') + return 0 + + +def get_cpuflag() -> Sequence[str]: + '''Return QEMU cpu option list suitable for host CPU''' + + try: + with open('/proc/cpuinfo', 'r') as f: + for line in f: + if line.startswith('flags'): + words = line.split() + if 'vmx' in words: + adtlog.debug( + 'Detected KVM capable Intel host CPU, ' + 'enabling nested KVM' + ) + return ['-cpu', 'kvm64,+vmx,+lahf_lm'] + elif 'svm' in words: # AMD kvm + adtlog.debug( + 'Detected KVM capable AMD host CPU, ' + 'enabling nested KVM' + ) + # FIXME: this should really be the one below + # for more reproducible testbeds, but nothing + # except -cpu host works + # return ['-cpu', 'kvm64,+svm,+lahf_lm'] + return ['-cpu', 'host'] + except IOError as e: + adtlog.warning( + 'Cannot read /proc/cpuinfo to detect CPU flags: %s' % e + ) + # fetching CPU flags isn't critical (only used to enable + # nested KVM), so don't fail here + + return [] + + +class QemuImage: + def __init__( + self, + file: str, + format: Optional[str] = None, + readonly: bool = False, + ) -> None: + self.file = file + self.overlay = None # type: Optional[str] + self.readonly = readonly + + if format is not None: + self.format = format + else: + info = json.loads( + VirtSubproc.check_exec([ + 'qemu-img', 'info', + '--output=json', + self.file, + ], outp=True, timeout=5) + ) + + if 'format' not in info: + VirtSubproc.bomb('Unable to determine format of %s' % self.file) + + self.format = str(info['format']) + + def __str__(self) -> str: + bits = [] # type: List[str] + + if self.overlay is None: + bits.append('file={}'.format(self.file)) + else: + bits.append('file={}'.format(self.overlay)) + bits.append('cache=unsafe') + + bits.append('if=virtio') + bits.append('discard=unmap') + bits.append('format={}'.format(self.format)) + + if self.readonly: + bits.append('readonly') + + return ','.join(bits) + + +class Qemu: + def __init__( + self, + images: Sequence[Union[QemuImage, str]], + qemu_command: str, + cpus: int = 1, + efi: bool = False, + overlay: bool = False, + overlay_dir: Optional[str] = None, + qemu_options: Sequence[str] = (), + ram_size: int = 1024, + workdir: Optional[str] = None, + ) -> None: + """ + Constructor. + + images: Disk images for the VM. The first image is assumed to be + the bootable, writable root filesystem, and we actually boot a + snapshot. The remaining images are assumed to be read-only. + qemu_command: qemu executable + + cpus: Number of vCPUs + efi: If true, boot using OVMF/AAVMF firmware (x86/ARM only) + overlay: If true, use a temporary overlay for first image + overlay_dir: Store writable overlays here (default: workdir) + qemu_options: Space-separated options for qemu + ram_size: Amount of RAM in MiB + workdir: Directory for temporary files (default: a random + subdirectory of $TMPDIR) + """ + + self.cpus = cpus + self.images = [] # type: List[QemuImage] + self.overlay_dir = overlay_dir + self.qemu_command = qemu_command + self.ram_size = ram_size + self.ssh_port = find_free_port(10022) + + if workdir is None: + workdir = tempfile.mkdtemp(prefix='autopkgtest-qemu.') + + self.workdir = workdir # type: Optional[str] + os.chmod(workdir, 0o755) + self.shareddir = os.path.join(workdir, 'shared') + os.mkdir(self.shareddir) + self.monitor_socket_path = os.path.join(workdir, 'monitor') + self.ttys0_socket_path = os.path.join(workdir, 'ttyS0') + self.ttys1_socket_path = os.path.join(workdir, 'ttyS1') + + for i, image in enumerate(images): + if isinstance(image, QemuImage): + self.images.append(image) + else: + assert isinstance(image, str) + + self.images.append( + QemuImage( + file=image, + format=None, + readonly=(i != 0), + ) + ) + + if overlay: + self.images[0].overlay = self.prepare_overlay(self.images[0]) + + if self.ssh_port: + adtlog.debug( + 'Forwarding local port %i to VM ssh port 22' % self.ssh_port + ) + nic_opt = ',hostfwd=tcp:127.0.0.1:%i-:22' % self.ssh_port + else: + nic_opt = '' + + # start QEMU + argv = [ + qemu_command, + '-m', str(ram_size), + '-smp', str(cpus), + '-nographic', + '-net', 'nic,model=virtio', + '-net', 'user' + nic_opt, + '-object', 'rng-random,filename=/dev/urandom,id=rng0', + '-device', 'virtio-rng-pci,rng=rng0,id=rng-device0', + '-monitor', 'unix:%s,server,nowait' % self.monitor_socket_path, + '-serial', 'unix:%s,server,nowait' % self.ttys0_socket_path, + '-serial', 'unix:%s,server,nowait' % self.ttys1_socket_path, + '-virtfs', + ( + 'local,id=autopkgtest,path=%s,security_model=none,' + 'mount_tag=autopkgtest' + ) % self.shareddir, + ] + + for i, image in enumerate(self.images): + argv.append('-drive') + argv.append('index=%d,%s' % (i, image)) + + if efi: + if 'qemu-system-x86_64' in qemu_command or \ + 'qemu-system-i386' in qemu_command: + code = '/usr/share/OVMF/OVMF_CODE.fd' + data = '/usr/share/OVMF/OVMF_VARS.fd' + elif 'qemu-system-aarch64' in qemu_command: + code = '/usr/share/AAVMF/AAVMF_CODE.fd' + data = '/usr/share/AAVMF/AAVMF_VARS.fd' + elif 'qemu-system-arm' in qemu_command: + code = '/usr/share/AAVMF/AAVMF32_CODE.fd' + data = '/usr/share/AAVMF/AAVMF32_VARS.fd' + else: + VirtSubproc.bomb('Unknown architecture for EFI boot') + + shutil.copy(data, '%s/efivars.fd' % workdir) + argv.append('-drive') + argv.append('if=pflash,format=raw,read-only,file=' + code) + argv.append('-drive') + argv.append( + 'if=pflash,format=raw,file=%s/efivars.fd' % workdir + ) + + if os.path.exists('/dev/kvm'): + argv.append('-enable-kvm') + # Enable nested KVM by default on x86_64 + if ( + os.uname()[4] == 'x86_64' and + self.qemu_command == 'qemu-system-x86_64' and + '-cpu' not in qemu_options + ): + argv += get_cpuflag() + + # pass through option to qemu + if qemu_options: + argv.extend(qemu_options) + + self.subprocess = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=sys.stderr, + stderr=subprocess.STDOUT, + ) # type: Optional[subprocess.Popen[bytes]] + + @staticmethod + def get_default_qemu_command( + uname_m: Optional[str] = None + ) -> str: + uname_to_qemu_suffix = {'i[3456]86$': 'i386', '^arm': 'arm'} + + if uname_m is None: + uname_m = os.uname()[4] + + for pattern, suffix in uname_to_qemu_suffix.items(): + if re.match(pattern, uname_m): + return 'qemu-system-' + suffix + break + else: + return 'qemu-system-' + uname_m + + @property + def monitor_socket(self): + return VirtSubproc.get_unix_socket(self.monitor_socket_path) + + @property + def ttys0_socket(self): + return VirtSubproc.get_unix_socket(self.ttys0_socket_path) + + @property + def ttys1_socket(self): + return VirtSubproc.get_unix_socket(self.ttys1_socket_path) + + def prepare_overlay( + self, + image: QemuImage, + ) -> str: + '''Generate a temporary overlay image''' + + # generate a temporary overlay + if self.overlay_dir is not None: + overlay = os.path.join( + self.overlay_dir, + os.path.basename(image.file) + '.overlay-%s' % time.time() + ) + else: + workdir = self.workdir + assert workdir is not None + overlay = os.path.join(workdir, 'overlay.img') + + adtlog.debug('Creating temporary overlay image in %s' % overlay) + VirtSubproc.check_exec( + [ + 'qemu-img', 'create', + '-f', 'qcow2', + '-F', image.format, + '-b', os.path.abspath(image.file), + overlay, + ], + outp=True, + timeout=300, + ) + return overlay + + def cleanup(self) -> Optional[int]: + ret = None + + if self.subprocess is not None: + self.subprocess.terminate() + ret = self.subprocess.wait() + self.subprocess = None + + if self.workdir is not None: + shutil.rmtree(self.workdir) + self.workdir = None + + return ret diff -Nru autopkgtest-5.15/lib/testdesc.py autopkgtest-5.16/lib/testdesc.py --- autopkgtest-5.15/lib/testdesc.py 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/lib/testdesc.py 2021-01-24 20:15:20.000000000 +0000 @@ -477,7 +477,7 @@ any_negative = True if _matches_architecture(testbed_arch, arch[1:]): raise Unsupported(name, "Test declares architecture as not " + - "supported: %s" % arch) + "supported: %s" % testbed_arch) if arch[0] != "!": any_positive = True @@ -494,7 +494,7 @@ if not arch_matched: raise Unsupported(name, "Test lists explicitly supported " + "architectures, but the current architecture " + - "%s isn't listed." % arch) + "%s isn't listed." % testbed_arch) def parse_debian_source(srcdir, testbed_caps, testbed_arch, control_path=None, diff -Nru autopkgtest-5.15/lib/VirtSubproc.py autopkgtest-5.16/lib/VirtSubproc.py --- autopkgtest-5.15/lib/VirtSubproc.py 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/lib/VirtSubproc.py 2021-01-24 20:15:20.000000000 +0000 @@ -31,9 +31,9 @@ import traceback import errno import time -import pipes import socket import shutil +import shlex import adtlog @@ -529,7 +529,7 @@ deststdout = devnull_read srcstdin = devnull_read - remfileq = pipes.quote(sd[iremote]) + remfileq = shlex.quote(sd[iremote]) if not dirsp: rune = 'cat %s%s' % ('><'[upp], remfileq) if upp: diff -Nru autopkgtest-5.15/Makefile autopkgtest-5.16/Makefile --- autopkgtest-5.15/Makefile 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/Makefile 2021-01-24 20:15:20.000000000 +0000 @@ -53,6 +53,7 @@ pythonfiles = lib/VirtSubproc.py \ lib/adtlog.py \ lib/autopkgtest_args.py \ + lib/autopkgtest_qemu.py \ lib/adt_testbed.py \ lib/adt_binaries.py \ lib/testdesc.py \ diff -Nru autopkgtest-5.15/runner/autopkgtest autopkgtest-5.16/runner/autopkgtest --- autopkgtest-5.15/runner/autopkgtest 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/runner/autopkgtest 2021-01-24 20:15:20.000000000 +0000 @@ -31,7 +31,7 @@ import shutil import atexit import json -import pipes +import shlex from debian import deb822 @@ -202,7 +202,7 @@ def create_testinfo(vserver_args): global testbed - info = {'virt_server': ' '.join([pipes.quote(w) for w in vserver_args])} + info = {'virt_server': ' '.join([shlex.quote(w) for w in vserver_args])} if testbed.initial_kernel_version: info['kernel_version'] = testbed.initial_kernel_version @@ -808,7 +808,10 @@ except Exception: errorcode = print_exception(sys.exc_info(), '') if tmp: - create_testinfo(vserver_args) + try: + create_testinfo(vserver_args) + except Exception: + errorcode = print_exception(sys.exc_info(), '') cleanup() sys.exit(errorcode) diff -Nru autopkgtest-5.15/setup-commands/ro-apt autopkgtest-5.16/setup-commands/ro-apt --- autopkgtest-5.15/setup-commands/ro-apt 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/setup-commands/ro-apt 2021-01-24 20:15:20.000000000 +0000 @@ -10,10 +10,10 @@ set -e M=$(mktemp --directory /run/ro-apt.XXXXX) -mount -t tmpfs tmpfs $M -cp -a /var/lib/dpkg/status /var/lib/dpkg/lock $M -cp -a /var/cache/apt $M/cache_apt -mount -o remount,ro $M -mount -o bind,ro $M/status /var/lib/dpkg/status -mount -o bind,ro $M/lock /var/lib/dpkg/lock -mount -o bind,ro $M/cache_apt /var/cache/apt +mount -t tmpfs tmpfs "$M" +cp -a /var/lib/dpkg/status /var/lib/dpkg/lock "$M" +cp -a /var/cache/apt "$M/cache_apt" +mount -o remount,ro "$M" +mount -o bind,ro "$M/status" /var/lib/dpkg/status +mount -o bind,ro "$M/lock" /var/lib/dpkg/lock +mount -o bind,ro "$M/cache_apt" /var/cache/apt diff -Nru autopkgtest-5.15/setup-commands/setup-testbed autopkgtest-5.16/setup-commands/setup-testbed --- autopkgtest-5.15/setup-commands/setup-testbed 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/setup-commands/setup-testbed 2021-01-24 20:15:20.000000000 +0000 @@ -82,7 +82,7 @@ fi # serial console for upstart -if [ -e "$root/etc/init/tty2.conf" -a ! -e "$root/etc/init/ttyS0.conf" ]; then +if [ -e "$root/etc/init/tty2.conf" ] && ! [ -e "$root/etc/init/ttyS0.conf" ]; then sed 's/tty2/ttyS0/g; s! *exec.*$!exec /sbin/getty -L ttyS0 115200 vt102!' \ "$root/etc/init/tty2.conf" > "$root/etc/init/ttyS0.conf" fi @@ -122,13 +122,14 @@ # set up apt sources if [ -e "$root/etc/os-release" ]; then - DISTRO_ID=`. "$root/etc/os-release" && echo "$ID" || echo INVALID` + # shellcheck disable=SC1090 + DISTRO_ID=$(. "$root/etc/os-release" && echo "$ID" || echo INVALID) fi if [ -z "${MIRROR:-}" ]; then - MIRROR=`awk '/^deb .*'"$DISTRO_ID"'/ { sub(/\[.*\]/, "", $0); print $2; exit }' "$root/etc/apt/sources.list"` + MIRROR=$(awk '/^deb .*'"$DISTRO_ID"'/ { sub(/\[.*\]/, "", $0); print $2; exit }' "$root/etc/apt/sources.list") fi if [ -z "${RELEASE:-}" ]; then - RELEASE=`awk '/^deb .*'"$DISTRO_ID"'/ { sub(/\[.*\]/, "", $0); print $3; exit }' "$root/etc/apt/sources.list"` + RELEASE=$(awk '/^deb .*'"$DISTRO_ID"'/ { sub(/\[.*\]/, "", $0); print $3; exit }' "$root/etc/apt/sources.list") fi if [ -n "${AUTOPKGTEST_KEEP_APT_SOURCES:-}" ]; then @@ -143,10 +144,14 @@ echo "$0: Attempting to set up Debian/Ubuntu apt sources automatically" >&2 if [ -z "$RELEASE" ]; then + # Deliberately not expanding $RELEASE here + # shellcheck disable=SC2016 echo 'Failed to auto-detect distribution release name; set $RELEASE explicitly' >&2 exit 1 fi if [ -z "$MIRROR" ]; then + # Deliberately not expanding $MIRROR here + # shellcheck disable=SC2016 echo 'Failed to auto-detect apt mirror; set $MIRROR explicitly' >&2 exit 1 fi @@ -207,7 +212,7 @@ if [ -n "$IFACE" ] ; then mkdir -p "$root/etc/network/interfaces.d" if ! grep -h -r "^[[:space:]]*auto.*$IFACE" "$root/etc/network/interfaces" "$root/etc/network/interfaces.d" | grep -qv 'auto[[:space:]]*lo'; then - printf "auto $IFACE\niface $IFACE inet dhcp\n" >> "$root/etc/network/interfaces.d/$IFACE" + printf 'auto %s\niface %s inet dhcp\n' "$IFACE" "$IFACE" >> "$root/etc/network/interfaces.d/$IFACE" fi fi fi @@ -221,10 +226,12 @@ # detect apt proxy on the host (in chroot mode) if [ "$root" != "/" ] && [ -z "$AUTOPKGTEST_APT_PROXY" ]; then - RES=`apt-config shell proxy Acquire::http::Proxy` + RES=$(apt-config shell proxy Acquire::http::Proxy) if [ -n "$RES" ]; then - eval $RES - if echo "$proxy" | egrep -q '(localhost|127\.0\.0\.[0-9]*)'; then + # evaluating $RES will set proxy, but shellcheck can't know that + proxy= + eval "$RES" + if echo "$proxy" | grep -E -q '(localhost|127\.0\.0\.[0-9]*)'; then AUTOPKGTEST_APT_PROXY=$(echo "$proxy" | sed -r "s#localhost|127\.0\.0\.[0-9]*#10.0.2.2#") elif [ -n "${proxy:-}" ]; then AUTOPKGTEST_APT_PROXY="$proxy" @@ -238,7 +245,7 @@ mv "$root/etc/resolv.conf" "$root/etc/resolv.conf.vmdebootstrap" fi cat /etc/resolv.conf > "$root/etc/resolv.conf" - trap "if [ -e '$root/etc/resolv.conf.vmdebootstrap' ]; then mv '$root/etc/resolv.conf.vmdebootstrap' '$root/etc/resolv.conf'; fi" EXIT INT QUIT PIPE + trap 'if [ -e "$root/etc/resolv.conf.vmdebootstrap" ]; then mv "$root/etc/resolv.conf.vmdebootstrap" "$root/etc/resolv.conf"; fi' EXIT INT QUIT PIPE fi if [ -z "${AUTOPKGTEST_IS_SETUP_COMMAND:-}" ]; then @@ -292,6 +299,8 @@ fi done if [ -n "$purge_list" ]; then + # Deliberately word-splitting $purge_list: + # shellcheck disable=SC2086 chroot "$root" eatmydata apt-get --auto-remove -y purge $purge_list || true fi @@ -311,11 +320,11 @@ fi if grep -q buntu "$root/etc/os-release" "$root/etc/lsb-release"; then - if ls $root/boot/vmlinu* >/dev/null 2>&1; then + if ls "$root"/boot/vmlinu* >/dev/null 2>&1; then # provides kmods like scsi_debug or mac80211_hwsim on Ubuntu chroot "$root" eatmydata apt-get install -y linux-generic < /dev/null else - if [ "$RELEASE" = precise -a "$ARCH" = armhf ]; then + if [ "$RELEASE" = precise ] && [ "$ARCH" = armhf ]; then # no linux-image-generic in precise/armhf yet chroot "$root" eatmydata apt-get install -y linux-headers-omap < /dev/null else diff -Nru autopkgtest-5.15/ssh-setup/SKELETON autopkgtest-5.16/ssh-setup/SKELETON --- autopkgtest-5.15/ssh-setup/SKELETON 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/ssh-setup/SKELETON 2021-01-24 20:15:20.000000000 +0000 @@ -65,15 +65,15 @@ case "$1" in open) - open $@;; + open "$@";; cleanup) - cleanup $@;; + cleanup "$@";; revert) - revert $@;; + revert "$@";; wait-reboot) - wait_reboot $@;; + wait_reboot "$@";; debug-failure) - debug_failure $@;; + debug_failure "$@";; '') echo "Needs to be called with command as first argument" >&2 exit 1 diff -Nru autopkgtest-5.15/tests/autopkgtest autopkgtest-5.16/tests/autopkgtest --- autopkgtest-5.15/tests/autopkgtest 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tests/autopkgtest 2021-01-24 20:15:20.000000000 +0000 @@ -4561,5 +4561,7 @@ if __name__ == '__main__': # Force encoding to UTF-8 even in non-UTF-8 locales. import io - sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding="UTF-8", line_buffering=True) + real_stdout = sys.stdout + assert isinstance(real_stdout, io.TextIOBase) + sys.stdout = io.TextIOWrapper(real_stdout.detach(), encoding="UTF-8", line_buffering=True) unittest.main(testRunner=unittest.TextTestRunner(stream=sys.stdout, verbosity=2)) diff -Nru autopkgtest-5.15/tests/autopkgtest_args autopkgtest-5.16/tests/autopkgtest_args --- autopkgtest-5.15/tests/autopkgtest_args 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tests/autopkgtest_args 2021-01-24 20:15:20.000000000 +0000 @@ -13,7 +13,7 @@ patch # pyflakes except ImportError: # fall back to separate package - from mock import patch + from mock import patch # type: ignore test_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(1, os.path.join(os.path.dirname(test_dir), 'lib')) diff -Nru autopkgtest-5.15/tests/mypy autopkgtest-5.16/tests/mypy --- autopkgtest-5.15/tests/mypy 1970-01-01 00:00:00.000000000 +0000 +++ autopkgtest-5.16/tests/mypy 2021-01-24 20:15:20.000000000 +0000 @@ -0,0 +1,48 @@ +#!/bin/sh +# Copyright © 2016-2020 Simon McVittie +# Copyright © 2018 Collabora Ltd. +# SPDX-License-Identifier: GPL-2+ + +set -e +set -u + +testdir="$(dirname "$(readlink -f "$0")")" +rootdir="$(dirname "$testdir")" + +export MYPYPATH="${PYTHONPATH:="${rootdir}/lib"}" + +i=0 +for file in \ + "$rootdir"/lib/*.py \ + "$rootdir"/runner/autopkgtest \ + "$rootdir"/tests/*.py \ + "$rootdir"/tests/autopkgtest \ + "$rootdir"/tests/autopkgtest_args \ + "$rootdir"/tests/qemu \ + "$rootdir"/tests/testdesc \ + "$rootdir"/tools/autopkgtest-build-qemu \ + "$rootdir"/tools/autopkgtest-buildvm-ubuntu-cloud \ + "$rootdir"/virt/autopkgtest-virt-chroot \ + "$rootdir"/virt/autopkgtest-virt-lxc \ + "$rootdir"/virt/autopkgtest-virt-lxd \ + "$rootdir"/virt/autopkgtest-virt-null \ + "$rootdir"/virt/autopkgtest-virt-qemu \ + "$rootdir"/virt/autopkgtest-virt-schroot \ + "$rootdir"/virt/autopkgtest-virt-ssh \ +; do + i=$((i + 1)) + if [ "x${MYPY:="$(command -v mypy || echo false)"}" = xfalse ]; then + echo "ok $i - $file # SKIP mypy not found" + elif "${MYPY}" \ + --python-executable="${PYTHON:=python3}" \ + --follow-imports=skip \ + --ignore-missing-imports \ + "$file"; then + echo "ok $i - $file" + else + echo "not ok $i - $file # TODO mypy issues reported" + fi +done +echo "1..$i" + +# vim:set sw=4 sts=4 et: diff -Nru autopkgtest-5.15/tests/pycodestyle autopkgtest-5.16/tests/pycodestyle --- autopkgtest-5.15/tests/pycodestyle 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tests/pycodestyle 2021-01-24 20:15:20.000000000 +0000 @@ -1,16 +1,26 @@ #!/bin/sh set -e -testdir="$(dirname $(readlink -f $0))" -rootdir="$(dirname $testdir)" -check=$(which pycodestyle || which pep8) +testdir="$(dirname "$(readlink -f "$0")")" +rootdir="$(dirname "$testdir")" +check=$(command -v pycodestyle || command -v pep8) status=0 -$check --ignore E402,E501,W504 $rootdir/lib/*.py $rootdir/tools/autopkgtest-buildvm-ubuntu-cloud || status=$? +"$check" --ignore E402,E501,W504 \ + "$rootdir"/lib/*.py \ + "$rootdir"/tools/autopkgtest-build-qemu \ + "$rootdir"/tools/autopkgtest-buildvm-ubuntu-cloud \ +|| status=$? for v in chroot null schroot lxc lxd qemu ssh; do - $check --ignore E501,E402,W504 $rootdir/virt/autopkgtest-virt-$v || status=$? + "$check" --ignore E501,E402,W504 "$rootdir/virt/autopkgtest-virt-$v" || status=$? done -$check --ignore E501,E402,W504 $rootdir/runner/autopkgtest $testdir/autopkgtest $testdir/testdesc $testdir/autopkgtest_args $testdir/*.py || status=$? +"$check" --ignore E501,E402,W504 \ + "$rootdir/runner/autopkgtest" \ + "$testdir/autopkgtest" \ + "$testdir/autopkgtest_args" \ + "$testdir/qemu" \ + "$testdir/testdesc" \ + "$testdir"/*.py || status=$? -exit $status +exit "$status" diff -Nru autopkgtest-5.15/tests/pyflakes autopkgtest-5.16/tests/pyflakes --- autopkgtest-5.15/tests/pyflakes 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tests/pyflakes 2021-01-24 20:15:20.000000000 +0000 @@ -7,18 +7,25 @@ # Author: Martin Pitt set -e -testdir="$(dirname $(readlink -f $0))" -rootdir="$(dirname $testdir)" +testdir="$(dirname "$(readlink -f "$0")")" +rootdir="$(dirname "$testdir")" if ! type pyflakes3 >/dev/null 2>&1; then echo "pyflakes3 not available, skipping" exit 0 fi -pyflakes3 $rootdir/lib $rootdir/runner/autopkgtest $testdir/autopkgtest \ - $testdir/testdesc $testdir/autopkgtest_args $testdir/*.py \ - $rootdir/tools/autopkgtest-buildvm-ubuntu-cloud +pyflakes3 \ + "$rootdir/lib" \ + "$rootdir/runner/autopkgtest" \ + "$testdir/autopkgtest" \ + "$testdir/autopkgtest_args" \ + "$testdir/qemu" \ + "$testdir/testdesc" \ + "$testdir"/*.py \ + "$rootdir/tools/autopkgtest-build-qemu" \ + "$rootdir/tools/autopkgtest-buildvm-ubuntu-cloud" for v in chroot null schroot lxc lxd qemu ssh; do - pyflakes3 $rootdir/virt/autopkgtest-virt-$v + pyflakes3 "$rootdir/virt/autopkgtest-virt-$v" done diff -Nru autopkgtest-5.15/tests/qemu autopkgtest-5.16/tests/qemu --- autopkgtest-5.15/tests/qemu 1970-01-01 00:00:00.000000000 +0000 +++ autopkgtest-5.16/tests/qemu 2021-01-24 20:15:20.000000000 +0000 @@ -0,0 +1,59 @@ +#!/usr/bin/python3 + +# This testsuite is part of autopkgtest. +# autopkgtest is a tool for testing Debian binary packages +# +# Copyright 2020 Simon McVittie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# See the file CREDITS for a full list of credits information (often +# installed as /usr/share/doc/autopkgtest/CREDITS). + +import os +import sys +import unittest + +test_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.dirname(test_dir) + +sys.path[:0] = [test_dir, os.path.join(root_dir, 'lib')] + +from autopkgtest_qemu import Qemu # noqa + + +class QemuTestCase(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_default_qemu_command(self) -> None: + get = Qemu.get_default_qemu_command + self.assertEqual(get('aarch64'), 'qemu-system-aarch64') + self.assertEqual(get('armv7l'), 'qemu-system-arm') + self.assertEqual(get('armv8l'), 'qemu-system-arm') + self.assertEqual(get('i686'), 'qemu-system-i386') + self.assertEqual(get('x86_64'), 'qemu-system-x86_64') + + +if __name__ == '__main__': + # Force encoding to UTF-8 even in non-UTF-8 locales. + import io + real_stdout = sys.stdout + assert isinstance(real_stdout, io.TextIOBase) + sys.stdout = io.TextIOWrapper(real_stdout.detach(), encoding="UTF-8", line_buffering=True) + unittest.main(testRunner=unittest.TextTestRunner(stream=sys.stdout, verbosity=2)) diff -Nru autopkgtest-5.15/tests/run-parallel autopkgtest-5.16/tests/run-parallel --- autopkgtest-5.15/tests/run-parallel 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tests/run-parallel 2021-01-24 20:15:20.000000000 +0000 @@ -1,21 +1,24 @@ #!/bin/sh # Run tests for different runners in parallel -MYDIR=$(dirname $0) +MYDIR=$(dirname "$0") # these are fast, run them first set -e -$MYDIR/pycodestyle -$MYDIR/pyflakes -$MYDIR/testdesc -$MYDIR/autopkgtest_args +"$MYDIR/mypy" +"$MYDIR/pycodestyle" +"$MYDIR/pyflakes" +"$MYDIR/qemu" +"$MYDIR/shellcheck" +"$MYDIR/testdesc" +"$MYDIR/autopkgtest_args" set +e # get sudo password early, to avoid asking for it in background jobs -[ `id -u` -eq 0 ] || sudo true +[ "$(id -u)" -eq 0 ] || sudo true -(OUT=$($MYDIR/autopkgtest QemuRunner 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit $rc) & -(OUT=$($MYDIR/autopkgtest LxcRunner SshRunnerNoScript SshRunnerWithScript 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit $rc) & -(OUT=$($MYDIR/autopkgtest NullRunner SchrootRunner SchrootClickRunner LxdRunner 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit $rc) & -(OUT=$(sudo $MYDIR/autopkgtest NullRunnerRoot ChrootRunner 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit $rc) & -for c in `seq 5`; do wait; done +(OUT=$("$MYDIR/autopkgtest" QemuRunner 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit "$rc") & +(OUT=$("$MYDIR/autopkgtest" LxcRunner SshRunnerNoScript SshRunnerWithScript 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit "$rc") & +(OUT=$("$MYDIR/autopkgtest" NullRunner SchrootRunner SchrootClickRunner LxdRunner 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit "$rc") & +(OUT=$(sudo "$MYDIR/autopkgtest" NullRunnerRoot ChrootRunner 2>&1) || rc=$?; echo "=== $c ==="; echo "$OUT"; exit "$rc") & +for c in $(seq 5); do wait; done diff -Nru autopkgtest-5.15/tests/shellcheck autopkgtest-5.16/tests/shellcheck --- autopkgtest-5.15/tests/shellcheck 1970-01-01 00:00:00.000000000 +0000 +++ autopkgtest-5.16/tests/shellcheck 2021-01-24 20:15:20.000000000 +0000 @@ -0,0 +1,45 @@ +#!/bin/sh +# Copyright © 2018-2020 Collabora Ltd +# SPDX-License-Identifier: GPL-2+ + +set -e +set -u + +testdir="$(dirname "$(readlink -f "$0")")" +rootdir="$(dirname "$testdir")" + +if ! command -v shellcheck >/dev/null 2>&1; then + echo "1..0 # SKIP shellcheck not available" + exit 0 +fi + +n=0 +for shell_script in \ + "$rootdir"/setup-commands/* \ + "$rootdir"/ssh-setup/* \ + "$rootdir"/tests/mypy \ + "$rootdir"/tests/pycodestyle \ + "$rootdir"/tests/pyflakes \ + "$rootdir"/tests/run-parallel \ + "$rootdir"/tests/shellcheck \ + "$rootdir"/tests/ssh-setup-lxd \ +; do + n=$((n + 1)) + + case "$shell_script" in + (*/ssh-setup/adb | */ssh-setup/maas | */ssh-setup/nova) + echo "ok $n - $shell_script # SKIP Someone who can test this needs to fix it" + continue + ;; + esac + + if shellcheck --shell=dash "$shell_script"; then + echo "ok $n - $shell_script" + else + echo "not ok $n # TODO - $shell_script" + fi +done + +echo "1..$n" + +# vim:set sw=4 sts=4 et: diff -Nru autopkgtest-5.15/tests/ssh-setup-lxd autopkgtest-5.16/tests/ssh-setup-lxd --- autopkgtest-5.15/tests/ssh-setup-lxd 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tests/ssh-setup-lxd 2021-01-24 20:15:20.000000000 +0000 @@ -27,36 +27,36 @@ # optional: port, options, capabilities open() { [ -z "$2" ] || IMAGE="$2" - if [ -z "$IMAGE}" ]; then + if [ -z "${IMAGE}" ]; then echo "ERROR: $0 needs to be called with image name" >&1 exit 1 fi - [ -n "$CONTAINER" ] || CONTAINER=`mktemp -u autopkgtest-test-XXX` + [ -n "$CONTAINER" ] || CONTAINER=$(mktemp -u autopkgtest-test-XXX) lxc launch --ephemeral "$IMAGE" "$CONTAINER" >/dev/null # wait for and parse IPv4 - while ! OUT=`lxc info $CONTAINER|grep 'eth0:.*inet[^6]'`; do + while ! OUT=$(lxc info "$CONTAINER"|grep 'eth0:.*inet[^6]'); do sleep 1 done IP=$(echo "$OUT" | grep -o '10\.[0-9]\+\.[0-9]\+\.[0-9]\+') # create user # password: python3 -c 'from crypt import *; print(crypt("autopkgtest", mksalt(METHOD_CRYPT)))' - lxc exec $CONTAINER -- useradd --password FJfXYBhFnX6xA --create-home $USER + lxc exec "$CONTAINER" -- useradd --password FJfXYBhFnX6xA --create-home "$USER" # install SSH - lxc exec $CONTAINER -- eatmydata apt-get install -y openssh-server >/dev/null 2>&1 + lxc exec "$CONTAINER" -- eatmydata apt-get install -y openssh-server >/dev/null 2>&1 if [ -n "$INSTALL_KEY" ]; then - key=`cat $HOME/.ssh/id_rsa.pub` - lxc exec $CONTAINER -- su -c "mkdir ~/.ssh; echo '$key' > ~/.ssh/authorized_keys" $USER + key=$(cat "$HOME/.ssh/id_rsa.pub") + lxc exec "$CONTAINER" -- su -c "mkdir ~/.ssh; echo '$key' > ~/.ssh/authorized_keys" "$USER" echo "identity=$HOME/.ssh/id_rsa" fi if [ -n "$ENABLE_SUDO" ]; then - lxc exec $CONTAINER -- sh -ec "echo '$USER ALL=(ALL) $ENABLE_SUDO' > /etc/sudoers.d/autopkgtest" + lxc exec "$CONTAINER" -- sh -ec "echo '$USER ALL=(ALL) $ENABLE_SUDO' > /etc/sudoers.d/autopkgtest" fi cat<" >&2 exit 1 fi - lxc delete --force $CONTAINER + lxc delete --force "$CONTAINER" } # parse options -eval set -- $(getopt -o "ksSn:I:c" -- "$@") +eval "set -- $(getopt -o "ksSn:I:c" -- "$@")" while true; do case "$1" in -k) @@ -109,11 +109,11 @@ case "$1" in open) - open $@;; + open "$@";; cleanup) - cleanup $@;; + cleanup "$@";; revert) - revert $@;; + revert "$@";; '') echo "Needs to be called with command as first argument" >&2 exit 1 diff -Nru autopkgtest-5.15/tests/testdesc autopkgtest-5.16/tests/testdesc --- autopkgtest-5.15/tests/testdesc 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tests/testdesc 2021-01-24 20:15:20.000000000 +0000 @@ -14,7 +14,7 @@ patch # pyflakes except ImportError: # fall back to separate package - from mock import patch + from mock import patch # type: ignore test_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(1, os.path.join(os.path.dirname(test_dir), 'lib')) @@ -935,5 +935,7 @@ if __name__ == '__main__': # Force encoding to UTF-8 even in non-UTF-8 locales. - sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding="UTF-8", line_buffering=True) + real_stdout = sys.stdout + assert isinstance(real_stdout, io.TextIOBase) + sys.stdout = io.TextIOWrapper(real_stdout.detach(), encoding="UTF-8", line_buffering=True) unittest.main(testRunner=unittest.TextTestRunner(stream=sys.stdout, verbosity=2)) diff -Nru autopkgtest-5.15/tools/autopkgtest-build-qemu autopkgtest-5.16/tools/autopkgtest-build-qemu --- autopkgtest-5.15/tools/autopkgtest-build-qemu 2020-10-26 20:27:25.000000000 +0000 +++ autopkgtest-5.16/tools/autopkgtest-build-qemu 2021-01-24 20:15:20.000000000 +0000 @@ -1,9 +1,12 @@ -#!/bin/sh +#!/usr/bin/python3 # autopkgtest-build-qemu is part of autopkgtest # autopkgtest is a tool for testing Debian binary packages # -# Copyright (C) Antonio Terceiro . +# Copyright (C) 2016-2020 Antonio Terceiro . +# Copyright (C) 2019 Sébastien Delafond +# Copyright (C) 2019-2020 Simon McVittie +# Copyright (C) 2020 Christian Kastner # # Build a QEMU image for using with autopkgtest # @@ -24,295 +27,385 @@ # See the file CREDITS for a full list of credits information (often # installed as /usr/share/doc/autopkgtest/CREDITS). -set -eu - -apt_proxy= -architecture= -mirror= -user_script= -size= - -usage () { - echo "usage: $0 [] [] [] [