diff -Nru livecd-rootfs-2.408.50/debian/changelog livecd-rootfs-2.408.54/debian/changelog --- livecd-rootfs-2.408.50/debian/changelog 2019-06-25 20:54:10.000000000 +0000 +++ livecd-rootfs-2.408.54/debian/changelog 2019-09-26 09:53:23.000000000 +0000 @@ -1,3 +1,35 @@ +livecd-rootfs (2.408.54) xenial; urgency=medium + + * Revert exclusion of makedev from buildd chroots, as it turned out not to + be the problem. Instead, fix up /dev/ptmx to be a character device node + rather than a symlink to /dev/pts/ptmx, in line with the discussion in + https://bugs.debian.org/817236; I think this is safer than + cherry-picking the fix to debootstrap at this point in a stable release + cycle (LP: #1844504). + + -- Colin Watson Thu, 26 Sep 2019 10:53:23 +0100 + +livecd-rootfs (2.408.53) xenial; urgency=medium + + * Fix exclusion of makedev from buildd chroots; debootstrap doesn't + respect --exclude for "Priority: required" packages, so we have to purge + makedev later (LP: #1844504). + + -- Colin Watson Tue, 24 Sep 2019 15:05:26 +0100 + +livecd-rootfs (2.408.52) xenial; urgency=medium + + * Exclude makedev from buildd chroots, since it was historically excluded + and apparently breaks some builds (e.g. mir; LP: #1844504). + + -- Colin Watson Wed, 18 Sep 2019 11:01:45 +0200 + +livecd-rootfs (2.408.51) xenial; urgency=medium + + * ubuntu-cpc: Only produce explicitly specified artifacts (LP: #1837254) + + -- Robert C Jennings Mon, 26 Aug 2019 16:32:41 -0500 + livecd-rootfs (2.408.50) xenial; urgency=medium * Actually, do not depend on snapd on powerpc as well. Snaps are not diff -Nru livecd-rootfs-2.408.50/live-build/auto/build livecd-rootfs-2.408.54/live-build/auto/build --- livecd-rootfs-2.408.50/live-build/auto/build 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/auto/build 2019-09-17 11:41:43.000000000 +0000 @@ -892,3 +892,8 @@ iptables -t nat -D OUTPUT -p tcp --dport 80 -m owner ! --uid-owner daemon \ -j REDIRECT --to 8080 fi + +case $PROJECT in + ubuntu-cpc) + config/hooks.d/remove-implicit-artifacts +esac diff -Nru livecd-rootfs-2.408.50/live-build/buildd/hooks/00-ptmx-chardev.chroot_early livecd-rootfs-2.408.54/live-build/buildd/hooks/00-ptmx-chardev.chroot_early --- livecd-rootfs-2.408.50/live-build/buildd/hooks/00-ptmx-chardev.chroot_early 1970-01-01 00:00:00.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/buildd/hooks/00-ptmx-chardev.chroot_early 2019-09-25 14:45:42.000000000 +0000 @@ -0,0 +1,19 @@ +#! /bin/sh +set -e + +# debootstrap 1.0.76 started creating /dev/ptmx as a symlink to +# /dev/pts/ptmx. Unfortunately, this doesn't work with sbuild, because it +# leaves the ptmxmode mount option at its default of 000, which causes +# builds to be unable to open /dev/pts/ptmx. To avoid this, debootstrap +# 1.0.89 switched to creating it as a device node where possible. See +# https://bugs.debian.org/817236 for details and analysis. +# +# xenial has a version of debootstrap in the range that contains this bug. +# It seems too risky to try to cherry-pick the debootstrap change in +# question in an SRU at this point; instead, just fix things up here for +# buildd images. + +if [ -h /dev/ptmx ]; then + mknod -m 666 /dev/ptmx.new c 5 2 + mv /dev/ptmx.new /dev/ptmx +fi diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/disk-image livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/disk-image --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/disk-image 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/disk-image 2019-09-17 11:41:43.000000000 +0000 @@ -1,3 +1,12 @@ base/disk-image.binary base/disk-image-uefi.binary base/disk-image-ppc64el.binary +provides livecd.ubuntu-cpc.ext4 +provides livecd.ubuntu-cpc.initrd-generic +provides livecd.ubuntu-cpc.initrd-generic-lpae +provides livecd.ubuntu-cpc.initrd-powerpc64-smp +provides livecd.ubuntu-cpc.kernel-generic +provides livecd.ubuntu-cpc.kernel-generic-lpae +provides livecd.ubuntu-cpc.kernel-powerpc64-smp +provides livecd.ubuntu-cpc.kernel-kvm +provides livecd.ubuntu-cpc.manifest diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 2019-09-17 11:41:43.000000000 +0000 @@ -1,2 +1,5 @@ depends disk-image base/qcow2-image.binary +provides livecd.ubuntu-cpc.disk1.img +provides livecd.ubuntu-cpc.uefi1.img +provides livecd.ubuntu-cpc.disk1.img.xz diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/root-dir livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/root-dir --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/root-dir 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/root-dir 2019-09-17 11:41:43.000000000 +0000 @@ -1,3 +1 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image base/root-tarball.binary diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/squashfs livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/squashfs --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/squashfs 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/squashfs 2019-09-17 11:41:43.000000000 +0000 @@ -1,4 +1,4 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image depends root-dir base/root-squashfs.binary +provides livecd.ubuntu-cpc.squashfs +provides livecd.ubuntu-cpc.squashfs.manifest diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/tarball livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/tarball --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/tarball 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/tarball 2019-09-17 11:41:43.000000000 +0000 @@ -1,4 +1,5 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image depends root-dir base/root-xz.binary +provides livecd.ubuntu-cpc.rootfs.tar.gz +provides livecd.ubuntu-cpc.rootfs.tar.xz +provides livecd.ubuntu-cpc.rootfs.manifest diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/vagrant livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/vagrant --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/vagrant 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/vagrant 2019-09-17 11:41:43.000000000 +0000 @@ -1,2 +1,3 @@ depends disk-image base/vagrant.binary +provides livecd.ubuntu-cpc.vagrant.box diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/vmdk livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/vmdk --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/vmdk 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/vmdk 2019-09-17 11:41:43.000000000 +0000 @@ -1,3 +1,6 @@ depends disk-image base/vmdk-image.binary base/vmdk-ova-image.binary +provides livecd.ubuntu-cpc.disk1.vmdk +provides livecd.ubuntu-cpc.uefi.vmdk +provides livecd.ubuntu-cpc.ova diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/wsl livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/wsl --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/base/series/wsl 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/base/series/wsl 2019-09-17 11:41:43.000000000 +0000 @@ -1,2 +1,4 @@ depends root-dir base/wsl-gz.binary +provides livecd.ubuntu-cpc.wsl.rootfs.tar.gz +provides livecd.ubuntu-cpc.wsl.rootfs.manifest diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/make-hooks livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/make-hooks --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/make-hooks 2019-06-25 20:51:39.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/make-hooks 2019-09-17 11:41:43.000000000 +0000 @@ -31,10 +31,19 @@ depends disk-image depends extra-settings extra/cloudB.binary + provides livecd.ubuntu-cpc.disk-kvm.img + provides livecd.ubuntu-cpc.disk-kvm.manifest Where "disk-image" and "extra-settings" may list scripts and dependencies which are to be processed before the script "extra/cloudB.binary" is called. +The "provides" directive defines a file that the hook creates; it can be +specified multiple times. The field is used by this script to generate a list +of output files created explicitly by the named image targets. The list is +saved to the "explicit_provides" file in the hooks output directory. In +the case of the "all" target this list would be empty. This list is +consumed by the "remove-implicit-artifacts" which is run at the end of the build. + ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate from scripts ending in ".binary". Even if you arrange them interleaved in your series files, the chroot scripts will be run before the binary scripts. @@ -74,6 +83,7 @@ self._quiet = quiet self._hooks_list = [] self._included = set() + self._provides = [] def reset(self): """Reset the internal state allowing instance to be reused for @@ -120,8 +130,9 @@ e.g. "vmdk" or "vagrant". """ self.collect_chroot_hooks() - self.collect_binary_hooks(image_sets) + self.collect_binary_hooks(image_sets, explicit_sets=True) self.create_symlinks() + self.create_explicit_provides() def collect_chroot_hooks(self): """Chroot hooks are numbered and not explicitly mentioned in series @@ -139,7 +150,7 @@ continue self._hooks_list.append(os.path.join("chroot", entry)) - def collect_binary_hooks(self, image_sets): + def collect_binary_hooks(self, image_sets, explicit_sets=False): """Search the series files for the given image_sets and parse them and their dependencies to generate a list of hook scripts to be run during image build. @@ -150,6 +161,11 @@ Populates the internal list of paths to hook scripts in the order in which the scripts are to be run. + + If "explicit_sets" is True, the files specified on lines starting + with "provides" will be added to self._provides to track explicit + output artifacts. This is only True for the initial images_sets + list, dependent image sets should set this to False. """ for image_set in image_sets: series_file = self.find_series_file(image_set) @@ -163,6 +179,7 @@ line = line.strip() if not line or line.startswith("#"): continue + m = re.match(r"^\s*depends\s+(\S+.*)$", line) if m: include_set = m.group(1) @@ -171,6 +188,13 @@ self._included.add(include_set) self.collect_binary_hooks([include_set,]) continue + + m = re.match(r"^\s*provides\s+(\S+.*)$", line) + if m: + if explicit_sets: + self._provides.append(m.group(1)) + continue + if not line in self._hooks_list: self._hooks_list.append(line) @@ -195,13 +219,32 @@ hook_basename = m.group("basename") linkname = ("%03d-" % counter) + hook_basename - linksrc = os.path.join(self._hooks_dir, linkname) - linkdest = os.path.relpath(os.path.join(self._script_dir, hook), + linkdest = os.path.join(self._hooks_dir, linkname) + linksrc = os.path.relpath(os.path.join(self._script_dir, hook), self._hooks_dir) if not self._quiet: print("[HOOK] %s => %s" % (linkname, hook)) - os.symlink(linkdest, linksrc) + os.symlink(linksrc, linkdest) + + def create_explicit_provides(self): + """ + Create a file named "explicit_provides" in self._script_dir + listing all files named on "provides" in the series files of + targets explicitly named by the user. The file is created but + left empty if there are no explict "provides" keywords in the + targets (this is the case for 'all') + """ + with open(os.path.join(self._script_dir, "explicit_provides"), "w", + encoding="utf-8") as fp: + empty = True + for provides in self._provides: + if not self._quiet: + print("[PROVIDES] %s" % provides) + fp.write("%s\n" % provides) + empty = False + if not empty: + fp.write('livecd.magic-proxy.log\n') def cli(self, args): """Command line interface to the hooks generator.""" diff -Nru livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts --- livecd-rootfs-2.408.50/live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts 1970-01-01 00:00:00.000000000 +0000 +++ livecd-rootfs-2.408.54/live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts 2019-09-17 11:41:43.000000000 +0000 @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +#-*- encoding: utf-8 -*- +""" +Remove output files not created by explicitly specified image targets + +This uses the 'explicit_provides' file generated by the 'make-hooks' +script. If the file is empty, all output will be saved. +""" +import glob +import os +import sys + +if __name__ == "__main__": + print('Running {}'.format(__file__)) + scriptname = os.path.basename(__file__) + explicit = set() + with open('./config/hooks.d/explicit_provides', 'r', + encoding='utf-8') as fp: + for filename in fp: + explicit.add(filename.rstrip()) + + if not explicit: + print('{}: explicit_provides is empty. ' + 'All binary output will be included'.format(scriptname)) + sys.exit(0) + + all = set(glob.glob('livecd.ubuntu-cpc.*')) + implicit = all - explicit + + print('{}: all artifacts considered: {}'.format(scriptname, all)) + print('{}: explict artifacts to keep: {}'.format(scriptname, explicit)) + print('{}: implicit artifacts to remove: {}'.format(scriptname, implicit)) + + for file in implicit: + if os.path.islink(file): + print('{}: unlinking {}'.format(scriptname, file)) + os.unlink(file) + elif os.path.isfile(file): + print('{}: removing {} ' + '{} bytes'.format(scriptname, file, os.stat(file).st_size)) + os.remove(file)