diff -Nru vmdb2-0.19/amd64-uefi.vmdb vmdb2-0.22/amd64-uefi.vmdb --- vmdb2-0.19/amd64-uefi.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/amd64-uefi.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,63 @@ +# This is a sample VMDB2 input file that specifies a simple system for +# an amd64 machine that boots with UEFI. + +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efi + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: / + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efi + + - mkfs: ext4 + partition: / + + - mount: / + + # Using the virtual-filesystems plugin here upsets qemu-debootstrap, + # which ends up unable to create /dev/fd within the chroot, causing + # the qemu-debootstrap phase to fail. Until we get to the bottom + # that, don't enable the plugin. + #- virtual-filesystems: / + + - unpack-rootfs: / + + - qemu-debootstrap: buster + arch: amd64 + mirror: http://deb.debian.org/debian + target: / + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-amd64 + fs-tag: / + unless: rootfs_unpacked + + - cache-rootfs: / + unless: rootfs_unpacked + + - chroot: / + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo amd64-uefi-vmdb2 > /etc/hostname + + - grub: uefi + tag: / + efi: efi diff -Nru vmdb2-0.19/amd64.vmdb vmdb2-0.22/amd64.vmdb --- vmdb2-0.19/amd64.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/amd64.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,56 @@ +# This is a sample VMDB2 input file to specify a simple +# system that boots on a 32-bit PC with BIOS. + +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: msdos + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 50% + tag: / + + - kpartx: "{{ output }}" + + - mkfs: ext4 + partition: / + + - mount: / + + - unpack-rootfs: / + + - qemu-debootstrap: buster + arch: amd64 + mirror: http://deb.debian.org/debian + target: / + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-amd64 + tag: / + unless: rootfs_unpacked + + - cache-rootfs: / + unless: rootfs_unpacked + + - chroot: / + shell: | + echo amd64 > /etc/hostname + apt -y install python3 parted + + - fstab: / + + - chroot: / + shell: | + sed -i 's,^root:[^:]*:,root::,' /etc/passwd + + - resize-rootfs: / + + - grub: bios + tag: / + quiet: false diff -Nru vmdb2-0.19/ansible.vmdb vmdb2-0.22/ansible.vmdb --- vmdb2-0.19/ansible.vmdb 2020-08-12 05:53:37.000000000 +0000 +++ vmdb2-0.22/ansible.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -36,7 +36,7 @@ - apt: install packages: - - python + - python3 fs-tag: / - cache-rootfs: / diff -Nru vmdb2-0.19/ansible.yml vmdb2-0.22/ansible.yml --- vmdb2-0.19/ansible.yml 2019-02-15 04:50:30.000000000 +0000 +++ vmdb2-0.22/ansible.yml 2021-01-16 15:14:04.000000000 +0000 @@ -4,4 +4,4 @@ - unset_root_password vars: hostname: pc - + ansible_python_interpreter: /usr/bin/python3 diff -Nru vmdb2-0.19/arm64-uefi.vmdb vmdb2-0.22/arm64-uefi.vmdb --- vmdb2-0.19/arm64-uefi.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/arm64-uefi.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,63 @@ +# This is a sample VMDB2 input file that specifies a simple system for +# an arm64 machine that boots with UEFI. + +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efi + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: / + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efi + + - mkfs: ext4 + partition: / + + - mount: / + + # Using the virtual-filesystems plugin here upsets qemu-debootstrap, + # which ends up unable to create /dev/fd within the chroot, causing + # the qemu-debootstrap phase to fail. Until we get to the bottom + # that, don't enable the plugin. + #- virtual-filesystems: / + + - unpack-rootfs: / + + - qemu-debootstrap: buster + arch: arm64 + mirror: http://deb.debian.org/debian + target: / + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-arm64 + fs-tag: / + unless: rootfs_unpacked + + - cache-rootfs: / + unless: rootfs_unpacked + + - chroot: / + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo arm64-uefi-vmdb2 > /etc/hostname + + - grub: uefi + tag: / + efi: efi diff -Nru vmdb2-0.19/armhf-uefi.vmdb vmdb2-0.22/armhf-uefi.vmdb --- vmdb2-0.19/armhf-uefi.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/armhf-uefi.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,64 @@ +# This is a sample VMDB2 input file that specifies a simple system for +# an armhf machine that boots with UEFI. + +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efi + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: / + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efi + + - mkfs: ext4 + partition: / + + - mount: / + + # Using the virtual-filesystems plugin here upsets qemu-debootstrap, + # which ends up unable to create /dev/fd within the chroot, causing + # the qemu-debootstrap phase to fail. Until we get to the bottom + # that, don't enable the plugin. + #- virtual-filesystems: / + + - unpack-rootfs: / + + - qemu-debootstrap: buster + arch: armhf + mirror: http://deb.debian.org/debian + target: / + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-armmp-lpae + fs-tag: / + unless: rootfs_unpacked + + - cache-rootfs: / + unless: rootfs_unpacked + + - chroot: / + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo armhf-uefi-vmdb2 > /etc/hostname + + - grub: uefi + tag: / + efi: efi + console: serial diff -Nru vmdb2-0.19/check vmdb2-0.22/check --- vmdb2-0.19/check 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/check 2021-01-16 15:14:04.000000000 +0000 @@ -3,11 +3,22 @@ set -eu set -o pipefail -echo Running unit tests ============================================ -python3 -m CoverageTestRunner --ignore-missing-from=without-tests yarns vmdb -echo +hideok=chronic +if ! command -v chronic > /dev/null +then + hideok= +fi +if [ "$#" -gt 0 ] && [ "$1" = -v ] +then + hideok= + shift +fi + +$hideok echo Running unit tests ============================================ +$hideok python3 -m CoverageTestRunner --ignore-missing-from=without-tests yarns vmdb +$hideok echo -echo Checking every plugin looks OK ============================================ +$hideok echo Checking every plugin looks OK ============================================ for file in vmdb/plugins/*.py do case "$file" in @@ -20,29 +31,29 @@ exit 1 fi done -echo +$hideok echo if command -v sp-codegen > /dev/null then - echo Running Subplot ============================================ - sp-codegen vmdb2.md -o test.py --run - echo + $hideok echo Running Subplot ============================================ + $hideok sp-codegen vmdb2.md -o test.py --run + $hideok echo fi -echo Running yarn tests ======================================== +$hideok echo Running yarn tests ======================================== if python3 -c 'import yarnutils' 2>/dev/null then - yarn \ + $hideok yarn \ --shell=python3 \ --shell-arg '' \ --shell-library yarns/lib.py \ --env "PYTHONPATH=$(pwd)/yarns" \ --cd-datadir \ yarns/*.yarn "$@" - echo + $hideok echo fi -echo Formatting docs ======================================== +$hideok echo Formatting docs ======================================== ./format.sh plugindocs() { @@ -58,7 +69,7 @@ } panic() { - echo "ERROR: $@" 1>&2 + echo "ERROR: $*" 1>&2 exit 1 } diff -Nru vmdb2-0.19/check-all vmdb2-0.22/check-all --- vmdb2-0.19/check-all 2020-08-12 05:53:37.000000000 +0000 +++ vmdb2-0.22/check-all 2021-01-16 15:14:04.000000000 +0000 @@ -1,33 +1,100 @@ -#!/bin/sh +#!/bin/bash set -eu -tarball="$1" +tarballdir="$(cd "$1" && pwd)" shift +amd64_tarball="$tarballdir/amd64.tar.gz" +i386_tarball="$tarballdir/i386.tar.gz" +arm64_tarball="$tarballdir/arm64.tar.gz" +armhf_tarball="$tarballdir/armhf.tar.gz" +ppc64el_tarball="$tarballdir/ppc64el.tar.gz" + tryit() { local image="$1" local yaml="$2" + local tarball="$3" + echo "Testing $yaml" - ./vmdb2 --rootfs-tarball "$dst/$tarball" \ + ./vmdb2 --rootfs-tarball "$tarball" \ --verbose \ - --log "$dst/check.log" \ + --log "$tarballdir/check.log" \ --output "$image" \ "$yaml" + rm -f "$image" echo } -src="$(dirname $0)" -dst="$(pwd)" +src="$(dirname "$0")" cd "$src" #tryit luks.img "$src/luks.vmdb" #tryit lvm2.img "$src/lvm2.vmdb"; exit -for x in "$@" pc uefi ansible smoke-pc smoke-uefi + +# This uses debootstrap, not qemu-debootstrap. Hence, it only works on amd64 +if [ -x /usr/bin/dpkg ] && [ "$(dpkg --print-architecture)" = "amd64" ] +then + bash -x ./smoke.sh "$amd64_tarball" + + for x in "$@" pc uefi ansible smoke-pc smoke-uefi + do + tryit "$tarballdir/$x.img" "$x.vmdb" "$amd64_tarball" + done +fi + +if [ -e /usr/share/OVMF/OVMF_VARS_4M.fd ] +then + bash -x ./smoke-amd64.sh "$amd64_tarball" +fi +tryit "$tarballdir/amd64.img" "amd64.vmdb" "$amd64_tarball" + +if [ -e /usr/share/OVMF/OVMF_VARS_4M.fd ] +then + tryit "$tarballdir/amd64-uefi.img" "amd64-uefi.vmdb" "$amd64_tarball" +fi + +if [ -e /usr/share/OVMF/OVMF32_VARS_4M.fd ] +then + bash -x ./smoke-i386.sh "$i386_tarball" +fi +tryit "$tarballdir/i386.img" "i386.vmdb" "$i386_tarball" + +if [ -e /usr/share/OVMF/OVMF32_VARS_4M.fd ] +then + tryit "$tarballdir/i386-uefi.img" "i386-uefi.vmdb" "$i386_tarball" +fi + + +# On amd64, qemu-user-static has occasionally been observed to segfault. This +# is being investigated further. +# [ 865.372027] show_signal_msg: 21 callbacks suppressed +# [ 865.372029] locale[37762]: segfault at 1eed310 ip 0000000000562f20 sp 00007ffe90ac9bf8 error 4 in qemu-aarch64-static[401000+3e3000] +# [ 865.372034] Code: 00 e9 94 7d 1c 00 0f 1f 40 00 64 83 2c 25 50 ff ff ff 01 74 05 c3 0f 1f 40 00 48 8d 3d c9 d9 7f 00 e9 e4 8a 1c 00 0f 1f 40 00 <64> 8b 04 25 50 ff ff ff 85 c0 0f 9f c0 c3 66 90 48 83 ec 08 64 8b +./smoke-arm64.sh "$arm64_tarball" + +for x in "$@" arm64-uefi do - tryit "$dst/$x.img" "$x.vmdb" + tryit "$tarballdir/$x.img" "$x.vmdb" "$arm64_tarball" done + +./smoke-armhf.sh "$armhf_tarball" + +for x in "$@" armhf-uefi +do + tryit "$tarballdir/$x.img" "$x.vmdb" "$armhf_tarball" +done + + +./smoke-ppc64el.sh "$ppc64el_tarball" + +for x in "$@" ppc64el +do + tryit "$tarballdir/$x.img" "$x.vmdb" "$ppc64el_tarball" +done + + echo All test images built successfully. diff -Nru vmdb2-0.19/debian/changelog vmdb2-0.22/debian/changelog --- vmdb2-0.19/debian/changelog 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/changelog 2021-01-19 18:04:32.000000000 +0000 @@ -1,3 +1,30 @@ +vmdb2 (0.22-1) unstable; urgency=medium + + [ Christian Kastner ] + * New upstream release. (Closes: #975303, #973467) + * Drop build dependencies python3-cliapp, help2man (no longer needed) + * Drop build dependencies debootstrap, kpartx, partx + This are not needed by the build time tests. + * Recommend ovmf, ovmf-ia32, qemu-efi-aarch64, qemu-efi-arm, qemu-slof, + qemu-system-x86, qemu-system-arm, qemu-system-ppc + * Re-enable build of HTML documentation + * Drop build of the PDF, and all of its dependencies + * Switch to using upstream-supplied man page + * Set Rules-Requires-Root: no + * Switch from debhelper (>= 12) to debhelper-compat (= 12) + * Use secure URI in debian/copyright + * Update Homepage + * Create debian/watch + * Add autopkgtest for module self-test + * Honor DEB_BUILD_OPTIONS=nodoc + * Remove build artifacts in clean target + * Bump Standards-Version to 4.5.1 + + [ Ryutaroh Matsumoto ] + * Recommend qemu-user-static. (Closes: #975024) + + -- Gunnar Wolf Tue, 19 Jan 2021 12:04:32 -0600 + vmdb2 (0.19-1) unstable; urgency=medium * Import new upstream version 0.19 diff -Nru vmdb2-0.19/debian/compat vmdb2-0.22/debian/compat --- vmdb2-0.19/debian/compat 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/compat 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -12 diff -Nru vmdb2-0.19/debian/control vmdb2-0.22/debian/control --- vmdb2-0.19/debian/control 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/control 2021-01-19 18:04:32.000000000 +0000 @@ -1,43 +1,37 @@ Source: vmdb2 -Homepage: https://vmdb2.liw.fi/ +Homepage: https://gitlab.com/larswirzenius/vmdb2 Maintainer: Gunnar Wolf Section: admin Priority: optional -Standards-Version: 4.5.0 +Rules-Requires-Root: no +Standards-Version: 4.5.1 Vcs-Browser: https://salsa.debian.org/debian/vmdb2/ Vcs-Git: https://salsa.debian.org/debian/vmdb2.git -Build-Depends: debhelper (>= 12), - python3-all, - python3-coverage-test-runner, - dh-python, - pandoc (>= 2.1.2~), - texlive-xetex, - lmodern, - fonts-freefont-ttf, - texlive-fonts-recommended, - python3-cliapp, - python3-jinja2, - cmdtest, - python3-yaml, - debootstrap, - qemu-utils, - parted, - kpartx, - help2man +Build-Depends: cmdtest, + debhelper-compat (= 12), + dh-python, + pandoc (>= 2.1.2~), + python3-all, + python3-coverage-test-runner, + python3-jinja2, + python3-yaml, Package: vmdb2 Architecture: all -Depends: ${python3:Depends}, ${misc:Depends}, - python3, - python3-cliapp, - python3-jinja2, - python3-yaml, - cmdtest, - debootstrap, - qemu-utils, - parted, - kpartx -Recommends: ansible +Depends: cmdtest, + debootstrap, + e2fsprogs, + kpartx, + parted, + python3, + python3-jinja2, + python3-yaml, + qemu-utils, + ${misc:Depends}, + ${python3:Depends}, +Recommends: ansible, + dosfstools, + qemu-user-static Description: creator of disk images with Debian installed vmdb2 will be a successor of vmdebootstrap. It will create disk images for virtual machines and real hardware, with partitioning, and a boot diff -Nru vmdb2-0.19/debian/copyright vmdb2-0.22/debian/copyright --- vmdb2-0.19/debian/copyright 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/copyright 2021-01-19 18:04:32.000000000 +0000 @@ -1,7 +1,7 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: vmdb2 Upstream-Contact: Lars Wirzenius -Source: http://git.liw.fi/vmdb2 +Source: https://gitlab.com/larswirzenius/vmdb2 Files: * Copyright: 2017-2018, Lars Wirzenius, Stuart Prescott diff -Nru vmdb2-0.19/debian/docs vmdb2-0.22/debian/docs --- vmdb2-0.19/debian/docs 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/docs 2021-01-19 18:04:32.000000000 +0000 @@ -1 +1,2 @@ -yarns/yarns.html +yarns/*.html +*.html diff -Nru vmdb2-0.19/debian/rules vmdb2-0.22/debian/rules --- vmdb2-0.19/debian/rules 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/rules 2021-01-19 18:04:32.000000000 +0000 @@ -5,6 +5,13 @@ %: dh $@ --with=python3 --buildsystem=pybuild +override_dh_auto_build: + dh_auto_build +ifeq (,$(filter nodoc, $(DEB_BUILD_OPTIONS) $(DEB_BUILD_PROFILES))) + make -C yarns + bash format.sh +endif + override_dh_auto_test: ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS))) ./check @@ -13,8 +20,6 @@ override_dh_installchangelogs: dh_installchangelogs -k NEWS -override_dh_installman: - [ ! -e vmdb2.1 ] - help2man -N ./vmdb2 -i debian/vmdb2.1.in > debian/vmdb2.1 - dh_installman - rm debian/vmdb2.1 +override_dh_auto_clean: + dh_auto_clean + rm -f build yarns/yarns.html vmdb2.html diff -Nru vmdb2-0.19/debian/tests/control vmdb2-0.22/debian/tests/control --- vmdb2-0.19/debian/tests/control 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/debian/tests/control 2021-01-19 18:04:32.000000000 +0000 @@ -0,0 +1,36 @@ +Test-Command: ./check +Depends: pandoc, python3-coverage-test-runner, vmdb2 +Restrictions: allow-stderr + +# Deactivated for now, as these succeed locally, but fail in autopkgtest +# environments, even with isolation-machine +#Test-Command: TERM=linux ./check-all "$AUTOPKGTEST_TMP" +# ; cp "$AUTOPKGTEST_TMP/check.log" "$AUTOPKGTEST_ARTIFACTS" +#Depends: ansible, +# binfmt-support, +# debootstrap, +# dosfstools, +# e2fsprogs, +# expect, +# ipxe-qemu, +# kpartx, +# ovmf, +# ovmf-ia32, +# pandoc, +# parted, +# python3-coverage-test-runner, +# qemu-efi-aarch64, +# qemu-efi-arm, +# qemu-slof, +# qemu-system-arm, +# qemu-system-ppc, +# qemu-system-x86, +# qemu-user-static, +# vmdb2, +#Restrictions: allow-stderr, needs-root, needs-internet, isolation-machine +# vmdb2 does not work under lxc, as we have the error from kpartx: +# /dev/mapper/control: open failed: Operation not permitted +# Failure to communicate with kernel device-mapper driver. +# Check that device-mapper is available in the kernel. +# Incompatible libdevmapper 1.02.173 (2020-08-09) and kernel driver (unknown version). +# device mapper prerequisites not met diff -Nru vmdb2-0.19/debian/vmdb2.1.in vmdb2-0.22/debian/vmdb2.1.in --- vmdb2-0.19/debian/vmdb2.1.in 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/vmdb2.1.in 1970-01-01 00:00:00.000000000 +0000 @@ -1,40 +0,0 @@ -[name] -vmdb2 \- Create a disk image with Debian installed - -[synopsis] -vmdb2 [ options ] SPECFILE - -[description] -vmdb2 reads a "specification file" that describes a disk image, and -builds the corresponding image. The specification file uses YAML -syntax, and contains a list of steps. Each step tells vmdb2 what to do -next. - -[examples] -To build a disk image: - -vmdb2 foo.vmdb - -[see also] -vmdebootstrap (1). - -See the vmdb2 manual and test suite for a full explanation of -specification files. This is installed in Debian systems under -the /usr/share/doc/vmdb2/ directory. - -[copyright] -Copyright 2017-2002 Lars Wirzenius - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - diff -Nru vmdb2-0.19/debian/vmdb2.docs vmdb2-0.22/debian/vmdb2.docs --- vmdb2-0.19/debian/vmdb2.docs 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/vmdb2.docs 2021-01-19 18:04:32.000000000 +0000 @@ -1,3 +1,2 @@ vmdb2.css vmdb2.html -vmdb2.pdf diff -Nru vmdb2-0.19/debian/vmdb2.manpages vmdb2-0.22/debian/vmdb2.manpages --- vmdb2-0.19/debian/vmdb2.manpages 2020-10-15 16:30:46.000000000 +0000 +++ vmdb2-0.22/debian/vmdb2.manpages 2021-01-19 18:04:32.000000000 +0000 @@ -1 +1 @@ -debian/vmdb2.1 +vmdb2.1 diff -Nru vmdb2-0.19/debian/watch vmdb2-0.22/debian/watch --- vmdb2-0.19/debian/watch 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/debian/watch 2021-01-19 18:04:32.000000000 +0000 @@ -0,0 +1,3 @@ +version=4 +opts=filenamemangle=s/.*\/archive\/vmdb2-(\d\S+)\/vmdb2.*\.tar\.gz/vmdb2-$1\.tar\.gz/g \ + https://gitlab.com/larswirzenius/vmdb2/tags?sort=updated_desc .*/archive/vmdb2-(\d\S+)/.*\.tar\.gz.* diff -Nru vmdb2-0.19/.gitignore vmdb2-0.22/.gitignore --- vmdb2-0.19/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/.gitignore 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,2 @@ +*.pyc +*.img diff -Nru vmdb2-0.19/i386-uefi.vmdb vmdb2-0.22/i386-uefi.vmdb --- vmdb2-0.19/i386-uefi.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/i386-uefi.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,63 @@ +# This is a sample VMDB2 input file that specifies a simple system for +# an i386 machine that boots with UEFI. + +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efi + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: / + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efi + + - mkfs: ext4 + partition: / + + - mount: / + + # Using the virtual-filesystems plugin here upsets qemu-debootstrap, + # which ends up unable to create /dev/fd within the chroot, causing + # the qemu-debootstrap phase to fail. Until we get to the bottom + # that, don't enable the plugin. + #- virtual-filesystems: / + + - unpack-rootfs: / + + - qemu-debootstrap: buster + arch: i386 + mirror: http://deb.debian.org/debian + target: / + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-686-pae + fs-tag: / + unless: rootfs_unpacked + + - cache-rootfs: / + unless: rootfs_unpacked + + - chroot: / + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo i386-uefi-vmdb2 > /etc/hostname + + - grub: uefi + tag: / + efi: efi diff -Nru vmdb2-0.19/i386.vmdb vmdb2-0.22/i386.vmdb --- vmdb2-0.19/i386.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/i386.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,56 @@ +# This is a sample VMDB2 input file to specify a simple +# system that boots on a 32-bit PC with BIOS. + +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: msdos + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 50% + tag: / + + - kpartx: "{{ output }}" + + - mkfs: ext4 + partition: / + + - mount: / + + - unpack-rootfs: / + + - qemu-debootstrap: buster + arch: i386 + mirror: http://deb.debian.org/debian + target: / + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-686-pae + tag: / + unless: rootfs_unpacked + + - cache-rootfs: / + unless: rootfs_unpacked + + - chroot: / + shell: | + echo i386 > /etc/hostname + apt -y install python3 parted + + - fstab: / + + - chroot: / + shell: | + sed -i 's,^root:[^:]*:,root::,' /etc/passwd + + - resize-rootfs: / + + - grub: bios + tag: / + quiet: false diff -Nru vmdb2-0.19/NEWS vmdb2-0.22/NEWS --- vmdb2-0.19/NEWS 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/NEWS 2021-01-16 15:14:04.000000000 +0000 @@ -1,6 +1,32 @@ NEWS for vmdb2, the Debian disk image builder ============================================================================= +Version 0.22, not yet released +----------------------------------------------------------------------------- + +* This release has a lot of changes from Ryutaroh Matsumoto and + Christian Kastner to fix and clean up things, and to add support for + building images for I385 (UEFI and BIOS), armhf+UEFI, and ppc64el. + + +Version 0.21, released 2020-12-18 +----------------------------------------------------------------------------- + +* The grub plugin is able to install the UEFI version of grub on arm64 + targets, patch from David Edmondson.[ + +* The documentation tile for the `resize_rootfs` step is now fixed. + + +Version 0.20, released 2020-10-18 +----------------------------------------------------------------------------- + +* All use of the Python `cliapp` library has been dropped from vmdb2 now. + +* New step `resize-rootfs` adds a systemd service to resize the root + file system to fit the actual drive on boot. + + Version 0.19, released 2020-09-16 ----------------------------------------------------------------------------- diff -Nru vmdb2-0.19/pc.vmdb vmdb2-0.22/pc.vmdb --- vmdb2-0.19/pc.vmdb 2020-08-12 05:53:37.000000000 +0000 +++ vmdb2-0.22/pc.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -1,5 +1,5 @@ # This is a sample VMDB2 input file to specify a simple -# system that boots on a PC with BIOS. +# system that boots on a 64-bit PC with BIOS. steps: - mkimg: "{{ output }}" @@ -11,7 +11,7 @@ - mkpart: primary device: "{{ output }}" start: 0% - end: 100% + end: 50% tag: / - kpartx: "{{ output }}" @@ -40,10 +40,16 @@ - chroot: / shell: | echo pc > /etc/hostname - apt -y install python + apt -y install python3 parted - fstab: / + - chroot: / + shell: | + sed -i 's,^root:[^:]*:,root::,' /etc/passwd + + - resize-rootfs: / + - grub: bios tag: / quiet: false diff -Nru vmdb2-0.19/ppc64el.vmdb vmdb2-0.22/ppc64el.vmdb --- vmdb2-0.19/ppc64el.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/ppc64el.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,61 @@ +# This is a sample VMDB2 input file that specifies a simple system for +# a ppc64el machine that boots with IEEE1275. + +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 10M + tag: prep + + - mkpart: primary + device: "{{ output }}" + start: 10M + end: 100% + tag: / + + - kpartx: "{{ output }}" + + - mkfs: ext4 + partition: / + + - mount: / + + # Using the virtual-filesystems plugin here upsets qemu-debootstrap, + # which ends up unable to create /dev/fd within the chroot, causing + # the qemu-debootstrap phase to fail. Until we get to the bottom + # that, don't enable the plugin. + #- virtual-filesystems: / + + - unpack-rootfs: / + + - qemu-debootstrap: buster + arch: ppc64el + mirror: http://deb.debian.org/debian + target: / + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-powerpc64le + fs-tag: / + unless: rootfs_unpacked + + - cache-rootfs: / + unless: rootfs_unpacked + + - chroot: / + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo ppc64el-vmdb2 > /etc/hostname + + - grub: ieee1275 + tag: / + prep: prep + console: serial diff -Nru vmdb2-0.19/README.md vmdb2-0.22/README.md --- vmdb2-0.19/README.md 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/README.md 2021-01-16 15:14:04.000000000 +0000 @@ -58,19 +58,25 @@ The following Python modules are used by vmdb2 (Debian package names in brackets). -* cliapp [`python3-cliapp`] * jinja2 [`python3-jinja2`] * yaml [`python3-yaml`] +If UEFI booting is to be used, firmware is needed for the following +architectures (Debian package names providing it in brackets): + +* amd64 [`ovmf`] +* arm64 [`qemu-efi-aarch64`] +* arm [`qemu-efi-arm`] +* i386 [`ovmf-ia32`] + Dependencies for smoke.sh ----------------------------------------------------------------------------- -You probably need the following installed to run the smoke test: +You probably need the following installed to run the smoke tests: - git - python3-coverage-test-runner -- python3-cliapp - python3-jinja2 - cmdtest 0.31 or later - qemu-utils @@ -79,6 +85,10 @@ - debootstrap - expect - qemu-system +- ovmf +- ovmf-ia32 +- qemu-efi-aarch64 +- qemu-efi-arm Tutorial diff -Nru vmdb2-0.19/setup.py vmdb2-0.22/setup.py --- vmdb2-0.19/setup.py 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/setup.py 2021-01-16 15:14:04.000000000 +0000 @@ -15,57 +15,13 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from distutils.core import setup, Extension -from distutils.cmd import Command -from distutils.command.build import build -from distutils.command.clean import clean -import os +from distutils.core import setup import glob -import subprocess import vmdb -class Build(build): - def run(self): - build.run(self) - self.build_manpage("vmdb2", "") - self.format_yarns() - - def build_manpage(self, program, lang): - return - # building manpage fails on by unstable CI worker for mysterious - # reasons, will re-enable later - print("building manpage for %s (lang=%s)" % (program, lang)) - self.generate_troff(program, lang) - self.format_man_as_txt(program) - - def generate_troff(self, program, lang): - with open("%s.1%s" % (program, lang), "w") as f: - subprocess.check_call( - [ - "python3", - program, - "--generate-manpage=%s.1%s.in" % (program, lang), - "--output=%s.1" % program, - ], - stdout=f, - ) - - def format_man_as_txt(self, program): - env = dict(os.environ) - env["MANWIDTH"] = "80" - with open("%s.1.txt" % program, "w") as f: - subprocess.check_call( - ["man", "-l", "%s.1" % program], ["col", "-b"], stdout=f, env=env - ) - - def format_yarns(self): - print("building yarns") - subprocess.check_call(["make", "-C", "yarns"]) - - setup( name="vmdb2", version=vmdb.__version__, @@ -76,5 +32,4 @@ scripts=["vmdb2"], packages=["vmdb", "vmdb.plugins"], data_files=[("share/man/man1", glob.glob("*.1"))], - cmdclass={"build": Build}, ) diff -Nru vmdb2-0.19/smoke-amd64.sh vmdb2-0.22/smoke-amd64.sh --- vmdb2-0.19/smoke-amd64.sh 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-amd64.sh 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,8 @@ +#!/bin/sh + +set -eu + +tarball="$1" +shift + +yarn smoke-amd64.yarn --env ROOTFS_TARBALL="$tarball" "$@" diff -Nru vmdb2-0.19/smoke-amd64.vmdb vmdb2-0.22/smoke-amd64.vmdb --- vmdb2-0.19/smoke-amd64.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-amd64.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,72 @@ +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: msdos + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 1M + end: 10M + tag: unused + + - mkpart: primary + device: "{{ output }}" + start: 10M + end: 100% + tag: rootfs + + - kpartx: "{{ output }}" + + - mkfs: ext4 + partition: rootfs + label: smoke + + - mount: rootfs + + - create-dir: /smoke-dir + perm: 0777 + uid: 1000 + gid: 1000 + + - create-file: /smoke-create-file.txt + contents: No smoke here. + perm: 0777 + uid: 1000 + gid: 1000 + + - copy-file: /smoke-copy-file.txt + src: smoke-copy-file.txt + perm: 0777 + uid: 1000 + gid: 1000 + + - unpack-rootfs: rootfs + + - qemu-debootstrap: buster + arch: amd64 + mirror: http://deb.debian.org/debian + keyring: /usr/share/keyrings/debian-archive-keyring.gpg + target: rootfs + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-amd64 + tag: rootfs + unless: rootfs_unpacked + + - cache-rootfs: rootfs + unless: rootfs_unpacked + + - chroot: rootfs + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo amd64-vmdb2 > /etc/hostname + + - fstab: rootfs + + - grub: bios + tag: rootfs + console: serial diff -Nru vmdb2-0.19/smoke-amd64.yarn vmdb2-0.22/smoke-amd64.yarn --- vmdb2-0.19/smoke-amd64.yarn 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-amd64.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,101 @@ +# Smoke test vmdb2-built images + +This yarn file builds a basic image, and runs it under Qemu. The image +is configured to have a serial console enabled, and the scenario below +will log in as root (no password), and command the virtual machine to +power off. If this works, the image is at least minimally functional, +so the "smoke test" passes. + +More elaborate testing can be added, but smoke testing is enough for +now. + +To run this yarn file, you need to run it as root (since it needs to +build an image as root), and you need to run on a system where +qemu-system-x86_64 is installed. Additionally, you need to add the +following option to yarn: + EXAMPLE + --env ROOTFS_TARBALL=/path/to/rootfs/tarball + +This means vmdb2 will use the given tarball when creating an image, +and if the tarball doesn't exist yet, it will create it. This makes +testing multiple time much faster. + + SCENARIO smoke test amd64 image + WHEN user runs vmdb smoke-amd64.vmdb --output smoke-amd64.img + THEN user can BIOS boot smoke-amd64.img and power it off from root shell + + WHEN user runs vmdb smoke-uefi-amd64.vmdb --output smoke-uefi-amd64.img + THEN user can UEFI boot smoke-uefi-amd64.img and power it off from root shell + + IMPLEMENTS WHEN user runs vmdb (\S+) --output (\S+) + "$SRCDIR/vmdb2" "$SRCDIR/$MATCH_1" \ + --output "$DATADIR/$MATCH_2" \ + --log "$DATADIR/vmdb.log" \ + --verbose \ + --rootfs-tarball "$ROOTFS_TARBALL" + + IMPLEMENTS THEN user can BIOS boot (\S+) and power it off from root shell + cd "$DATADIR" + img="$MATCH_1" + cat << EOF > run.sh + qemu-system-x86_64 -drive file="$img",format=raw -m 1024 -nographic + EOF + chmod a+rx run.sh + cat << EOF > expect.txt + set timeout 300 + proc abort {} { + puts "ERROR ERROR\n" + exit 1 + } + spawn ./run.sh + expect "login: " + send "root\n" + expect "# " + send "poweroff\r" + set timeout 5 + expect { + "reboot: Power down" {puts poweroffing\n} + eof abort + timeout abort + } + expect eof + wait + EOF + expect -d expect.txt > expect.out + + IMPLEMENTS THEN user can UEFI boot (\S+) and power it off from root shell + cd "$DATADIR" + img="$MATCH_1" + cat << EOF > run.sh + cp /usr/share/OVMF/OVMF_VARS_4M.fd . + qemu-system-x86_64 \ + -machine q35 \ + -cpu max \ + -m 1024 \ + -drive if=pflash,format=raw,unit=0,file=/usr/share/OVMF/OVMF_CODE_4M.fd,readonly=on \ + -drive if=pflash,format=raw,unit=1,file=OVMF_VARS_4M.fd \ + -drive format=raw,file="$img" \ + -nographic + EOF + chmod a+rx run.sh + cat << EOF > expect.txt + set timeout 600 + proc abort {} { + puts "ERROR ERROR\n" + exit 1 + } + spawn ./run.sh + expect "login: " + send "root\n" + expect "# " + send "poweroff\r" + set timeout 5 + expect { + "reboot: Power down" {puts poweroffing\n} + eof abort + timeout abort + } + expect eof + wait + EOF + expect -d expect.txt > expect.out diff -Nru vmdb2-0.19/smoke-arm64.sh vmdb2-0.22/smoke-arm64.sh --- vmdb2-0.19/smoke-arm64.sh 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-arm64.sh 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,8 @@ +#!/bin/sh + +set -eu + +tarball="$1" +shift + +yarn smoke-arm64.yarn --env ROOTFS_TARBALL="$tarball" "$@" diff -Nru vmdb2-0.19/smoke-arm64.yarn vmdb2-0.22/smoke-arm64.yarn --- vmdb2-0.19/smoke-arm64.yarn 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-arm64.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,69 @@ +# Smoke test vmdb2-built images + +This yarn file builds a basic image, and runs it under Qemu. The image +is configured to have a serial console enabled, and the scenario below +will log in as root (no password), and command the virtual machine to +power off. If this works, the image is at least minimally functional, +so the "smoke test" passes. + +More elaborate testing can be added, but smoke testing is enough for +now. + +To run this yarn file, you need to run it as root (since it needs to +build an image as root), and you need to run on a system where +qemu-system-aarch64 is installed. Additionally, you need to add the +following option to yarn: + + EXAMPLE + --env ROOTFS_TARBALL=/path/to/rootfs/tarball + +This means vmdb2 will use the given tarball when creating an image, +and if the tarball doesn't exist yet, it will create it. This makes +testing multiple time much faster. + + SCENARIO smoke test arm64 UEFI image + WHEN user runs vmdb smoke-uefi-arm64.vmdb --output smoke-uefi-arm64.img + THEN user can UEFI boot smoke-uefi-arm64.img and power it off from root shell + + IMPLEMENTS WHEN user runs vmdb (\S+) --output (\S+) + "$SRCDIR/vmdb2" "$SRCDIR/$MATCH_1" \ + --output "$DATADIR/$MATCH_2" \ + --log "$SRCDIR/vmdb.log" \ + --verbose \ + --rootfs-tarball "$ROOTFS_TARBALL" + + IMPLEMENTS THEN user can UEFI boot (\S+) and power it off from root shell + cd "$DATADIR" + img="$MATCH_1" + cat << EOF > run.sh + cp /usr/share/AAVMF/AAVMF_VARS.fd . + qemu-system-aarch64 \ + -machine virt \ + -cpu max \ + -m 1024 \ + -drive if=pflash,format=raw,unit=0,file=/usr/share/AAVMF/AAVMF_CODE.fd,readonly=on \ + -drive if=pflash,format=raw,unit=1,file=AAVMF_VARS.fd \ + -drive format=raw,file="$img" \ + -nographic + EOF + chmod a+rx run.sh + cat << EOF > expect.txt + set timeout 300 + proc abort {} { + puts "ERROR ERROR\n" + exit 1 + } + spawn ./run.sh + expect "login: " + send "root\n" + expect "# " + send "poweroff\r" + set timeout 5 + expect { + "reboot: Power down" {puts poweroffing\n} + eof abort + timeout abort + } + expect eof + EOF + expect -d expect.txt > expect.out diff -Nru vmdb2-0.19/smoke-armhf.sh vmdb2-0.22/smoke-armhf.sh --- vmdb2-0.19/smoke-armhf.sh 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-armhf.sh 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,8 @@ +#!/bin/sh + +set -eu + +tarball="$1" +shift + +yarn smoke-armhf.yarn --env ROOTFS_TARBALL="$tarball" "$@" diff -Nru vmdb2-0.19/smoke-armhf.yarn vmdb2-0.22/smoke-armhf.yarn --- vmdb2-0.19/smoke-armhf.yarn 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-armhf.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,70 @@ +# Smoke test vmdb2-built images + +This yarn file builds a basic image, and runs it under Qemu. The image +is configured to have a serial console enabled, and the scenario below +will log in as root (no password), and command the virtual machine to +power off. If this works, the image is at least minimally functional, +so the "smoke test" passes. + +More elaborate testing can be added, but smoke testing is enough for +now. + +To run this yarn file, you need to run it as root (since it needs to +build an image as root), and you need to run on a system where +qemu-system-aarch64 is installed. Additionally, you need to add the +following option to yarn: + + EXAMPLE + --env ROOTFS_TARBALL=/path/to/rootfs/tarball + +This means vmdb2 will use the given tarball when creating an image, +and if the tarball doesn't exist yet, it will create it. This makes +testing multiple time much faster. + + SCENARIO smoke test armhf UEFI image + WHEN user runs vmdb smoke-uefi-armhf.vmdb --output smoke-uefi-armhf.img + THEN user can UEFI boot smoke-uefi-armhf.img and power it off from root shell + + IMPLEMENTS WHEN user runs vmdb (\S+) --output (\S+) + "$SRCDIR/vmdb2" "$SRCDIR/$MATCH_1" \ + --output "$DATADIR/$MATCH_2" \ + --log "$DATADIR/vmdb.log" \ + --verbose \ + --rootfs-tarball "$ROOTFS_TARBALL" + + IMPLEMENTS THEN user can UEFI boot (\S+) and power it off from root shell + cd "$DATADIR" + img="$MATCH_1" + cat << EOF > run.sh + cp /usr/share/AAVMF/AAVMF32_VARS.fd . + qemu-system-arm \ + -machine virt \ + -cpu max \ + -m 1024 \ + -drive if=pflash,format=raw,unit=0,file=/usr/share/AAVMF/AAVMF32_CODE.fd,readonly=on \ + -drive if=pflash,format=raw,unit=1,file=AAVMF32_VARS.fd \ + -drive format=raw,file="$img" \ + -nographic + EOF + chmod a+rx run.sh + cat << EOF > expect.txt + set timeout 600 + proc abort {} { + puts "ERROR ERROR\n" + exit 1 + } + spawn ./run.sh + expect "login: " + send "root\n" + expect "# " + send "poweroff\r" + set timeout 5 + expect { + "reboot: Power down" {puts poweroffing\n} + eof abort + timeout abort + } + expect eof + wait + EOF + expect -d expect.txt > expect.out diff -Nru vmdb2-0.19/smoke-i386.sh vmdb2-0.22/smoke-i386.sh --- vmdb2-0.19/smoke-i386.sh 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-i386.sh 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,8 @@ +#!/bin/sh + +set -eu + +tarball="$1" +shift + +yarn smoke-i386.yarn --env ROOTFS_TARBALL="$tarball" "$@" diff -Nru vmdb2-0.19/smoke-i386.vmdb vmdb2-0.22/smoke-i386.vmdb --- vmdb2-0.19/smoke-i386.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-i386.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,72 @@ +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: msdos + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 1M + end: 10M + tag: unused + + - mkpart: primary + device: "{{ output }}" + start: 10M + end: 100% + tag: rootfs + + - kpartx: "{{ output }}" + + - mkfs: ext4 + partition: rootfs + label: smoke + + - mount: rootfs + + - create-dir: /smoke-dir + perm: 0777 + uid: 1000 + gid: 1000 + + - create-file: /smoke-create-file.txt + contents: No smoke here. + perm: 0777 + uid: 1000 + gid: 1000 + + - copy-file: /smoke-copy-file.txt + src: smoke-copy-file.txt + perm: 0777 + uid: 1000 + gid: 1000 + + - unpack-rootfs: rootfs + + - qemu-debootstrap: buster + arch: i386 + mirror: http://deb.debian.org/debian + keyring: /usr/share/keyrings/debian-archive-keyring.gpg + target: rootfs + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-686-pae + tag: rootfs + unless: rootfs_unpacked + + - cache-rootfs: rootfs + unless: rootfs_unpacked + + - chroot: rootfs + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo pc-vmdb2 > /etc/hostname + + - fstab: rootfs + + - grub: bios + tag: rootfs + console: serial diff -Nru vmdb2-0.19/smoke-i386.yarn vmdb2-0.22/smoke-i386.yarn --- vmdb2-0.19/smoke-i386.yarn 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-i386.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,101 @@ +# Smoke test vmdb2-built images + +This yarn file builds a basic image, and runs it under Qemu. The image +is configured to have a serial console enabled, and the scenario below +will log in as root (no password), and command the virtual machine to +power off. If this works, the image is at least minimally functional, +so the "smoke test" passes. + +More elaborate testing can be added, but smoke testing is enough for +now. + +To run this yarn file, you need to run it as root (since it needs to +build an image as root), and you need to run on a system where +qemu-system-i386 is installed. Additionally, you need to add the +following option to yarn: + EXAMPLE + --env ROOTFS_TARBALL=/path/to/rootfs/tarball + +This means vmdb2 will use the given tarball when creating an image, +and if the tarball doesn't exist yet, it will create it. This makes +testing multiple time much faster. + + SCENARIO smoke test i386 image + WHEN user runs vmdb smoke-i386.vmdb --output smoke-i386.img + THEN user can BIOS boot smoke-i386.img and power it off from root shell + + WHEN user runs vmdb smoke-uefi-i386.vmdb --output smoke-uefi-i386.img + THEN user can UEFI boot smoke-uefi-i386.img and power it off from root shell + + IMPLEMENTS WHEN user runs vmdb (\S+) --output (\S+) + "$SRCDIR/vmdb2" "$SRCDIR/$MATCH_1" \ + --output "$DATADIR/$MATCH_2" \ + --log "$DATADIR/vmdb.log" \ + --verbose \ + --rootfs-tarball "$ROOTFS_TARBALL" + + IMPLEMENTS THEN user can BIOS boot (\S+) and power it off from root shell + cd "$DATADIR" + img="$MATCH_1" + cat << EOF > run.sh + qemu-system-i386 -drive file="$img",format=raw -m 1024 -nographic + EOF + chmod a+rx run.sh + cat << EOF > expect.txt + set timeout 300 + proc abort {} { + puts "ERROR ERROR\n" + exit 1 + } + spawn ./run.sh + expect "login: " + send "root\n" + expect "# " + send "poweroff\r" + set timeout 5 + expect { + "reboot: Power down" {puts poweroffing\n} + eof abort + timeout abort + } + expect eof + wait + EOF + expect -d expect.txt > expect.out + + IMPLEMENTS THEN user can UEFI boot (\S+) and power it off from root shell + cd "$DATADIR" + img="$MATCH_1" + cat << EOF > run.sh + cp /usr/share/OVMF/OVMF32_VARS_4M.fd . + qemu-system-i386 \ + -machine q35 \ + -cpu max \ + -m 1024 \ + -drive if=pflash,format=raw,unit=0,file=/usr/share/OVMF/OVMF32_CODE_4M.secboot.fd,readonly=on \ + -drive if=pflash,format=raw,unit=1,file=OVMF32_VARS_4M.fd \ + -drive format=raw,file="$img" \ + -nographic + EOF + chmod a+rx run.sh + cat << EOF > expect.txt + set timeout 600 + proc abort {} { + puts "ERROR ERROR\n" + exit 1 + } + spawn ./run.sh + expect "login: " + send "root\n" + expect "# " + send "poweroff\r" + set timeout 5 + expect { + "reboot: Power down" {puts poweroffing\n} + eof abort + timeout abort + } + expect eof + wait + EOF + expect -d expect.txt > expect.out diff -Nru vmdb2-0.19/smoke-ppc64el.sh vmdb2-0.22/smoke-ppc64el.sh --- vmdb2-0.19/smoke-ppc64el.sh 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-ppc64el.sh 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,8 @@ +#!/bin/sh + +set -eu + +tarball="$1" +shift + +yarn smoke-ppc64el.yarn --env ROOTFS_TARBALL="$tarball" "$@" diff -Nru vmdb2-0.19/smoke-ppc64el.vmdb vmdb2-0.22/smoke-ppc64el.vmdb --- vmdb2-0.19/smoke-ppc64el.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-ppc64el.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,73 @@ +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 10M + tag: prep + + - mkpart: primary + device: "{{ output }}" + start: 10M + end: 100% + tag: rootfs + + - kpartx: "{{ output }}" + + - mkfs: ext4 + partition: rootfs + label: smoke + + - mount: rootfs + + - create-dir: /smoke-dir + perm: 0777 + uid: 1000 + gid: 1000 + + - create-file: /smoke-create-file.txt + contents: No smoke here. + perm: 0777 + uid: 1000 + gid: 1000 + + - copy-file: /smoke-copy-file.txt + src: smoke-copy-file.txt + perm: 0777 + uid: 1000 + gid: 1000 + + - unpack-rootfs: rootfs + + - qemu-debootstrap: buster + arch: ppc64el + mirror: http://deb.debian.org/debian + keyring: /usr/share/keyrings/debian-archive-keyring.gpg + target: rootfs + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-powerpc64le + tag: rootfs + unless: rootfs_unpacked + + - cache-rootfs: rootfs + unless: rootfs_unpacked + + - chroot: rootfs + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo ppc64el-vmdb2 > /etc/hostname + + - fstab: rootfs + + - grub: ieee1275 + tag: rootfs + prep: prep + console: serial diff -Nru vmdb2-0.19/smoke-ppc64el.yarn vmdb2-0.22/smoke-ppc64el.yarn --- vmdb2-0.19/smoke-ppc64el.yarn 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-ppc64el.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,65 @@ +# Smoke test vmdb2-built images + +This yarn file builds a basic image, and runs it under Qemu. The image +is configured to have a serial console enabled, and the scenario below +will log in as root (no password), and command the virtual machine to +power off. If this works, the image is at least minimally functional, +so the "smoke test" passes. + +More elaborate testing can be added, but smoke testing is enough for +now. + +To run this yarn file, you need to run it as root (since it needs to +build an image as root), and you need to run on a system where +qemu-system-ppc64le is installed. Additionally, you need to add the +following option to yarn: + + EXAMPLE + --env ROOTFS_TARBALL=/path/to/rootfs/tarball + +This means vmdb2 will use the given tarball when creating an image, +and if the tarball doesn't exist yet, it will create it. This makes +testing multiple time much faster. + + SCENARIO smoke test ppc64el IEEE1275 image + WHEN user runs vmdb smoke-ppc64el.vmdb --output smoke-ppc64el.img + THEN user can IEEE1275 boot smoke-ppc64el.img and power it off from root shell + + IMPLEMENTS WHEN user runs vmdb (\S+) --output (\S+) + "$SRCDIR/vmdb2" "$SRCDIR/$MATCH_1" \ + --output "$DATADIR/$MATCH_2" \ + --log "$DATADIR/vmdb.log" \ + --verbose \ + --rootfs-tarball "$ROOTFS_TARBALL" + + IMPLEMENTS THEN user can IEEE1275 boot (\S+) and power it off from root shell + cd "$DATADIR" + img="$MATCH_1" + cat << EOF > run.sh + qemu-system-ppc64le \ + -m 1024 \ + -nographic \ + "$img" + EOF + chmod a+rx run.sh + cat << EOF > expect.txt + set timeout 600 + proc abort {} { + puts "ERROR ERROR\n" + exit 1 + } + spawn ./run.sh + expect "login: " + send "root\n" + expect "# " + send "poweroff\r" + set timeout 30 + expect { + "reboot: Power down" {puts poweroffing\n} + eof abort + timeout abort + } + expect eof + wait + EOF + expect -d expect.txt > expect.out diff -Nru vmdb2-0.19/smoke-uefi-amd64.vmdb vmdb2-0.22/smoke-uefi-amd64.vmdb --- vmdb2-0.19/smoke-uefi-amd64.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-uefi-amd64.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,76 @@ +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efifs + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: rootfs + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efifs + + - mkfs: ext4 + partition: rootfs + label: smoke + + - mount: rootfs + + - create-dir: /smoke-dir + perm: 0777 + uid: 1000 + gid: 1000 + + - create-file: /smoke-create-file.txt + contents: No smoke here. + perm: 0777 + uid: 1000 + gid: 1000 + + - copy-file: /smoke-copy-file.txt + src: smoke-copy-file.txt + perm: 0777 + uid: 1000 + gid: 1000 + + - unpack-rootfs: rootfs + + - qemu-debootstrap: buster + arch: amd64 + mirror: http://deb.debian.org/debian + keyring: /usr/share/keyrings/debian-archive-keyring.gpg + target: rootfs + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-amd64 + tag: rootfs + unless: rootfs_unpacked + + - cache-rootfs: rootfs + unless: rootfs_unpacked + + - chroot: rootfs + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo amd64-vmdb2 > /etc/hostname + + - fstab: rootfs + + - grub: uefi + tag: rootfs + efi: efifs + console: serial diff -Nru vmdb2-0.19/smoke-uefi-arm64.vmdb vmdb2-0.22/smoke-uefi-arm64.vmdb --- vmdb2-0.19/smoke-uefi-arm64.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-uefi-arm64.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,76 @@ +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efifs + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: rootfs + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efifs + + - mkfs: ext4 + partition: rootfs + label: smoke + + - mount: rootfs + + - create-dir: /smoke-dir + perm: 0777 + uid: 1000 + gid: 1000 + + - create-file: /smoke-create-file.txt + contents: No smoke here. + perm: 0777 + uid: 1000 + gid: 1000 + + - copy-file: /smoke-copy-file.txt + src: smoke-copy-file.txt + perm: 0777 + uid: 1000 + gid: 1000 + + - unpack-rootfs: rootfs + + - qemu-debootstrap: buster + arch: arm64 + mirror: http://deb.debian.org/debian + keyring: /usr/share/keyrings/debian-archive-keyring.gpg + target: rootfs + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-arm64 + tag: rootfs + unless: rootfs_unpacked + + - cache-rootfs: rootfs + unless: rootfs_unpacked + + - chroot: rootfs + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo arm64-vmdb2 > /etc/hostname + + - fstab: rootfs + + - grub: uefi + tag: rootfs + efi: efifs + console: serial diff -Nru vmdb2-0.19/smoke-uefi-armhf.vmdb vmdb2-0.22/smoke-uefi-armhf.vmdb --- vmdb2-0.19/smoke-uefi-armhf.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-uefi-armhf.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,76 @@ +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efifs + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: rootfs + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efifs + + - mkfs: ext4 + partition: rootfs + label: smoke + + - mount: rootfs + + - create-dir: /smoke-dir + perm: 0777 + uid: 1000 + gid: 1000 + + - create-file: /smoke-create-file.txt + contents: No smoke here. + perm: 0777 + uid: 1000 + gid: 1000 + + - copy-file: /smoke-copy-file.txt + src: smoke-copy-file.txt + perm: 0777 + uid: 1000 + gid: 1000 + + - unpack-rootfs: rootfs + + - qemu-debootstrap: buster + arch: armhf + mirror: http://deb.debian.org/debian + keyring: /usr/share/keyrings/debian-archive-keyring.gpg + target: rootfs + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-armmp-lpae + tag: rootfs + unless: rootfs_unpacked + + - cache-rootfs: rootfs + unless: rootfs_unpacked + + - chroot: rootfs + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo armhf-vmdb2 > /etc/hostname + + - fstab: rootfs + + - grub: uefi + tag: rootfs + efi: efifs + console: serial diff -Nru vmdb2-0.19/smoke-uefi-i386.vmdb vmdb2-0.22/smoke-uefi-i386.vmdb --- vmdb2-0.19/smoke-uefi-i386.vmdb 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/smoke-uefi-i386.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,76 @@ +steps: + - mkimg: "{{ output }}" + size: 4G + + - mklabel: gpt + device: "{{ output }}" + + - mkpart: primary + device: "{{ output }}" + start: 0% + end: 1G + tag: efifs + + - mkpart: primary + device: "{{ output }}" + start: 1G + end: 100% + tag: rootfs + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: efifs + + - mkfs: ext4 + partition: rootfs + label: smoke + + - mount: rootfs + + - create-dir: /smoke-dir + perm: 0777 + uid: 1000 + gid: 1000 + + - create-file: /smoke-create-file.txt + contents: No smoke here. + perm: 0777 + uid: 1000 + gid: 1000 + + - copy-file: /smoke-copy-file.txt + src: smoke-copy-file.txt + perm: 0777 + uid: 1000 + gid: 1000 + + - unpack-rootfs: rootfs + + - qemu-debootstrap: buster + arch: i386 + mirror: http://deb.debian.org/debian + keyring: /usr/share/keyrings/debian-archive-keyring.gpg + target: rootfs + unless: rootfs_unpacked + + - apt: install + packages: + - linux-image-686-pae + tag: rootfs + unless: rootfs_unpacked + + - cache-rootfs: rootfs + unless: rootfs_unpacked + + - chroot: rootfs + shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd + echo i386-vmdb2 > /etc/hostname + + - fstab: rootfs + + - grub: uefi + tag: rootfs + efi: efifs + console: serial diff -Nru vmdb2-0.19/smoke.yarn vmdb2-0.22/smoke.yarn --- vmdb2-0.19/smoke.yarn 2019-02-15 04:50:31.000000000 +0000 +++ vmdb2-0.22/smoke.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -11,8 +11,8 @@ To run this yarn file, you need to run it as root (since it needs to build an image as root), and you need to run on an amd64 system (and -thus build an amd64 image). Additionally, you need to add the folloing -option to yarn: +thus build an amd64 image). Additionally, you need to add the +following option to yarn: EXAMPLE --env ROOTFS_TARBALL=/path/to/rootfs/tarball @@ -29,7 +29,7 @@ THEN user can UEFI boot smoke-uefi.img and power it off from root shell IMPLEMENTS WHEN user runs vmdb (\S+) --output (\S+) - "$SRCDIR/vmdb2" --no-default-config "$SRCDIR/$MATCH_1" \ + "$SRCDIR/vmdb2" "$SRCDIR/$MATCH_1" \ --output "$DATADIR/$MATCH_2" \ --log "$DATADIR/vmdb.log" \ --verbose \ diff -Nru vmdb2-0.19/uefi.vmdb vmdb2-0.22/uefi.vmdb --- vmdb2-0.19/uefi.vmdb 2020-08-12 05:53:37.000000000 +0000 +++ vmdb2-0.22/uefi.vmdb 2021-01-16 15:14:04.000000000 +0000 @@ -1,5 +1,5 @@ # This is a sample VMDB2 input file that specifies a simple system for -# a PC that boots with UEFI. +# a 64-bit PC that boots with UEFI. steps: - mkimg: "{{ output }}" @@ -50,6 +50,7 @@ - chroot: / shell: | + sed -i '/^root:[^:]*:/s//root::/' /etc/passwd echo uefi-vmdb2 > /etc/hostname - fstab: / diff -Nru vmdb2-0.19/vmdb/app.py vmdb2-0.22/vmdb/app.py --- vmdb2-0.19/vmdb/app.py 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/vmdb/app.py 2021-01-16 15:14:04.000000000 +0000 @@ -16,44 +16,141 @@ # =*= License: GPL-3+ =*= +import argparse import logging import os import sys -import cliapp - import vmdb -class Vmdb2(cliapp.Application): - def add_settings(self): - self.settings.string( - ["image"], - "use existing image file/device FILE (use --output to create new file)", - metavar="FILE", +class Vmdb2: + def __init__(self, version): + self._version = version + + def run(self): + args = self.parse_command_line() + vmdb.set_verbose_progress(args.verbose) + tvars = self.template_vars_from_args(args) + + cmd = None + if args.version: + cmd = VersionCommand(self._version) + elif args.image or args.output: + if args.image: + cmd = ReuseImageCommand(args.image, args.specfile, args.rootfs_tarball) + else: + cmd = NewImageCommand(args.output, args.specfile, args.rootfs_tarball) + builder = cmd.builder() + builder.add_template_vars(tvars) + + if cmd is None: + sys.exit("I don't know what to do, re-run with --help") + + if args.log: + self.setup_logging(args.log) + self.log_startup() + try: + cmd.run() + except Exception as e: + logging.error(f"ERROR: {e}", exc_info=True) + sys.exit(f"ERROR: {e}\n") + logging.debug("Ending, all OK") + + def parse_command_line(self): + p = argparse.ArgumentParser( + description="build disk images with Debian installed" ) - self.settings.boolean(["verbose", "v"], "verbose output") + p.add_argument("--image", metavar="FILE") + p.add_argument("--output", metavar="FILE") + p.add_argument("--rootfs-tarball", metavar="FILE") + p.add_argument("-v", "--verbose", action="store_true") + p.add_argument("--log") + p.add_argument("--version", action="store_true") + p.add_argument("specfile") - def setup(self): - self.step_runners = vmdb.StepRunnerList() - plugindir = os.path.join(os.path.dirname(vmdb.__file__), "plugins") - for klass in vmdb.find_plugins(plugindir, "Plugin"): - klass(self).enable() + return p.parse_args() + + def setup_logging(self, filename): + fmt = "%(asctime)s %(levelname)s %(message)s" + datefmt = "%Y-%m-%d %H:%M:%S" + formatter = logging.Formatter(fmt, datefmt) + + handler = logging.FileHandler(filename) + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + def log_startup(self): + logging.info(f"Starting vmdb2 version {self._version}") + + def template_vars_from_args(self, args): + return { + "image": args.image, + "output": args.output, + "rootfs_tarball": args.rootfs_tarball, + } + + +class Command: + def run(self): + raise NotImplementedError() + + +class VersionCommand(Command): + def __init__(self, version): + self.version = version + + def run(self): + sys.stdout.write("{}\n".format(self.version)) - def process_args(self, args): - if len(args) != 1: - sys.exit("No image specification was given on the command line.") - vmdb.set_verbose_progress(self.settings["verbose"]) +class NewImageCommand(Command): + def __init__(self, filename, specfile, tarball): + self._builder = ImageBuilder(filename, specfile, tarball) - spec = self.load_spec_file(args[0]) + def builder(self): + return self._builder + + def run(self): + # FIXME: create file + self._builder.build() + + +class ReuseImageCommand(Command): + def __init__(self, filename, specfile, tarball): + self._builder = ImageBuilder(filename, specfile, tarball) + + def builder(self): + return self._builder + + def run(self): + self._builder.build() + + +class ImageBuilder: + def __init__(self, filename, specfile, tarball): + self._filename = filename + self._specfile = specfile + self._tarball = tarball + self._tvars = {} + + def add_template_vars(self, tvars): + self._tvars.update(tvars) + + def build(self): + spec = self.load_spec_file(self._specfile) state = vmdb.State() state.tags = vmdb.Tags() - params = self.create_template_vars(state) - steps = spec.get_steps(params) + state.arch = vmdb.runcmd(["dpkg", "--print-architecture"]).decode("UTF-8").strip() + self.add_template_vars(state.as_dict()) + steps = spec.get_steps(self._tvars) # Check that we have step runners for each step + self.load_step_runners() for step in steps: self.step_runners.find(step) @@ -66,7 +163,7 @@ vmdb.progress("All went fine.") if core_meltdown: - logging.error("An error occurred, exiting with non-zero exit code") + logging.error("An error occurred, exiting") sys.exit(1) def load_spec_file(self, filename): @@ -80,6 +177,12 @@ spec.load_file(f) return spec + def load_step_runners(self): + self.step_runners = vmdb.StepRunnerList() + plugindir = os.path.join(os.path.dirname(vmdb.__file__), "plugins") + for klass in vmdb.find_plugins(plugindir, "Plugin"): + klass(self).enable() + def run_steps(self, steps, state): return self.run_steps_helper(steps, state, "Running step: %r", "run", False) @@ -91,6 +194,7 @@ def run_steps_helper(self, steps, state, msg, method_name, keep_going): core_meltdown = False steps_taken = [] + settings = {"rootfs-tarball": self._tarball} even_if_skipped = method_name + "_even_if_skipped" for step in steps: @@ -98,7 +202,7 @@ logging.info(msg, step) steps_taken.append(step) runner = self.step_runners.find(step) - if runner.skip(step, self.settings, state): + if runner.skip(step, settings, state): logging.info("Skipping as requested by unless") method_names = [even_if_skipped] else: @@ -113,7 +217,7 @@ values = runner.get_values(step) for method in methods: logging.info("Calling %s", method) - method(values, self.settings, state) + method(values, settings, state) except KeyError as e: vmdb.error("Key error: %s" % str(e)) vmdb.error(repr(e)) @@ -128,10 +232,3 @@ break return steps_taken, core_meltdown - - def create_template_vars(self, state): - vars = dict() - for key in self.settings: - vars[key] = self.settings[key] - vars.update(state.as_dict()) - return vars diff -Nru vmdb2-0.19/vmdb/plugins/cache_rootfs_plugin.py vmdb2-0.22/vmdb/plugins/cache_rootfs_plugin.py --- vmdb2-0.19/vmdb/plugins/cache_rootfs_plugin.py 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/cache_rootfs_plugin.py 2021-01-16 15:14:04.000000000 +0000 @@ -23,12 +23,6 @@ class CacheRootFSPlugin(vmdb.Plugin): def enable(self): - self.app.settings.string( - ["rootfs-tarball"], - "store rootfs cache tar archives in FILE", - metavar="FILE", - ) - self.app.step_runners.add(MakeCacheStepRunner()) diff -Nru vmdb2-0.19/vmdb/plugins/grub.mdwn vmdb2-0.22/vmdb/plugins/grub.mdwn --- vmdb2-0.19/vmdb/plugins/grub.mdwn 2020-08-12 05:53:37.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/grub.mdwn 2021-01-16 15:14:04.000000000 +0000 @@ -1,22 +1,26 @@ Step: grub ----------------------------------------------------------------------------- -Install the GRUB bootloader to the image. Works on a PC, for -traditional BIOS booting or modern UEFI booting. Does not (yet?) -support Secure Boot. +Install the GRUB bootloader to the image. Works on a PC for +traditional BIOS booting, PC and ARM machines for modern +UEFI booting, and PowerPC machines for IEEE1275 booting. Supports Secure Boot +for amd64 UEFI. Warning: This is the least robust part of vmdb2. Step keys: * `grub` — REQUIRED; value MUST be one of `uefi` and `bios`, for - a UEFI or a BIOS boot, respectively. (FIXME: these are valid for a - PC; not sure what other archs require, if grub even works there.) + a UEFI or a BIOS boot, respectively. Only PC systems support the + `bios` option. * `tag` — REQUIRED; value is the tag for the root filesystem. * `efi` — REQUIRED for UEFI; value is the tag for the EFI - filesystem. + partition. + +* `prep` — REQUIRED for IEEE1275; value is the tag for the + PReP partition. * `console` — OPTIONAL; set to `serial` to configure the image to use a serial console. @@ -35,13 +39,21 @@ - grub: bios tag: root -Same, but for UEFI: +Same, but for UEFI, assuming that a FAT32 filesystem exists on the +partition with tag `efi`: - grub: uefi tag: root efi: efi console: serial +Or for IEEE1275, assuming that a partition with tag `prep` exists: + + - grub: ieee1275 + tag: root + prep: prep + console: serial + Install to a real hard disk (named with the `--image` option): - grub: uefi diff -Nru vmdb2-0.19/vmdb/plugins/grub_plugin.py vmdb2-0.22/vmdb/plugins/grub_plugin.py --- vmdb2-0.19/vmdb/plugins/grub_plugin.py 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/grub_plugin.py 2021-01-16 15:14:04.000000000 +0000 @@ -51,9 +51,9 @@ # For cleanliness, we also undo any bind mounts into the chroot. Don't # want to leave them in case they cause trouble. # -# Note that this is currently rather strongly assuming that UEFI and -# the amd64 (a.k.a. x86_64) architecture are being used. These should -# probably not be hardcoded. Patch welcome. +# Note that this is currently assuming that UEFI and either the amd64 +# (a.k.a. x86_64) or arm64 (a.k.a. aarch64) architectures are being +# used. These should probably not be hardcoded. Patch welcome. # To use this plugin: write steps to create a root filesystem, and an # VFAT filesystem to be mounted as /boot/efi. Install Debian onto the @@ -88,6 +88,7 @@ "root-fs": "", "efi": "", "efi-part": "", + "prep": "", "console": "", "tag": "", "image-dev": "", @@ -102,9 +103,25 @@ self.install_uefi(values, settings, state) elif flavor == "bios": self.install_bios(values, settings, state) + elif flavor == "ieee1275": + self.install_ieee1275(values, settings, state) else: raise Exception("Unknown GRUB flavor {}".format(flavor)) + def grub_uefi_variant(self, state): + variants = { + "amd64": ("grub-efi-amd64", "x86_64-efi"), + "i386": ("grub-efi-ia32", "i386-efi"), + "arm64": ("grub-efi-arm64", "arm64-efi"), + "armhf": ("grub-efi-arm", "arm-efi"), + } + try: + return variants[state.arch] + except KeyError: + raise Exception( + 'GRUB UEFI package and target for "{}" unknown'.format(state.arch) + ) + def install_uefi(self, values, settings, state): efi = values["efi"] or None efi_part = values["efi-part"] or None @@ -112,8 +129,7 @@ raise Exception('"efi" or "efi-part" required in UEFI GRUB installation') vmdb.progress("Installing GRUB for UEFI") - grub_package = "grub-efi-amd64" - grub_target = "x86_64-efi" + (grub_package, grub_target) = self.grub_uefi_variant(state) self.install_grub(values, settings, state, grub_package, grub_target) def install_bios(self, values, settings, state): @@ -122,6 +138,12 @@ grub_target = "i386-pc" self.install_grub(values, settings, state, grub_package, grub_target) + def install_ieee1275(self, values, settings, state): + vmdb.progress("Installing GRUB for IEEE1275") + grub_package = "grub-ieee1275" + grub_target = "powerpc-ieee1275" + self.install_grub(values, settings, state, grub_package, grub_target) + def install_grub(self, values, settings, state, grub_package, grub_target): console = values["console"] or None @@ -142,11 +164,23 @@ else: efi_dev = None + prep = values["prep"] or None + if prep: + prep_dev = state.tags.get_dev(prep) + else: + prep_dev = None + quiet = values["quiet"] - self.bind_mount_many(chroot, ["/dev", "/sys"], state) + self.bind_mount_many(chroot, ["/dev", "/sys", "/proc"], state) if efi_dev: + pn = efi_dev[-1] + vmdb.runcmd(["parted", "-s", image_dev, "set", pn, "esp", "on" ]) self.mount(chroot, efi_dev, "/boot/efi", state) + elif prep_dev: + pn = prep_dev[-1] + vmdb.runcmd(["parted", "-s", image_dev, "set", pn, "prep", "on" ]) + image_dev = prep_dev self.install_package(chroot, grub_package) kernel_params = [ @@ -157,9 +191,19 @@ "rw", ] if console == "serial": - kernel_params.extend( - ["loglevel=3", "console=tty0", "console=ttyS0,115200n8"] - ) + if 'ppc64' in state.arch: + kernel_params.extend( + ["loglevel=3", "console=tty0", "console=hvc0,115200n8"] + ) + elif 'arm' in state.arch: + kernel_params.extend( + ["loglevel=3", "console=tty0", "console=ttyAMA0,115200n8"] + ) + else: + kernel_params.extend( + ["loglevel=3", "console=tty0", "console=ttyS0,115200n8"] + ) + if quiet: kernel_params.extend( [ diff -Nru vmdb2-0.19/vmdb/plugins/mkimg_plugin.py vmdb2-0.22/vmdb/plugins/mkimg_plugin.py --- vmdb2-0.19/vmdb/plugins/mkimg_plugin.py 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/mkimg_plugin.py 2021-01-16 15:14:04.000000000 +0000 @@ -22,7 +22,6 @@ class MkimgPlugin(vmdb.Plugin): def enable(self): self.app.step_runners.add(MkimgStepRunner()) - self.app.settings.bytesize(["size"], "size of output image", default="1GiB") class MkimgStepRunner(vmdb.StepRunnerInterface): diff -Nru vmdb2-0.19/vmdb/plugins/mkpart_plugin.py vmdb2-0.22/vmdb/plugins/mkpart_plugin.py --- vmdb2-0.19/vmdb/plugins/mkpart_plugin.py 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/mkpart_plugin.py 2021-01-16 15:14:04.000000000 +0000 @@ -50,7 +50,7 @@ device = os.path.realpath(device) orig = self.list_partitions(device) - vmdb.runcmd(["parted", "-s", device, "mkpart", part_type, fs_type, start, end]) + vmdb.runcmd(["parted", "-s", device, "--", "mkpart", part_type, fs_type, start, end]) new = self.list_partitions(device) diff = self.diff_partitions(orig, new) diff -Nru vmdb2-0.19/vmdb/plugins/qemudebootstrap_plugin.py vmdb2-0.22/vmdb/plugins/qemudebootstrap_plugin.py --- vmdb2-0.19/vmdb/plugins/qemudebootstrap_plugin.py 2020-10-15 16:15:49.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/qemudebootstrap_plugin.py 2021-01-16 15:14:04.000000000 +0000 @@ -47,6 +47,10 @@ components = values["components"] if not (suite and tag and target and mirror and arch): raise Exception("missing arg for qemu-debootstrap step") + # Update the state with the target architecture declared here + # in order that later stages can find it and behave + # appropriately. + state.arch = arch if keyring: vmdb.runcmd( [ @@ -84,4 +88,5 @@ def run_even_if_skipped(self, values, settings, state): tag = values["target"] target = state.tags.get_builder_mount_point(tag) + state.arch = values["arch"] vmdb.runcmd_chroot(target, ["apt-get", "update"]) diff -Nru vmdb2-0.19/vmdb/plugins/resize.mdwn vmdb2-0.22/vmdb/plugins/resize.mdwn --- vmdb2-0.19/vmdb/plugins/resize.mdwn 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/resize.mdwn 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,24 @@ +Step: resize-rootfs +----------------------------------------------------------------------------- + +Configure the system on the image so that it automatically resizes +itself to fill the actual disk, upon first boot. For this to work, the +root file system MUST be the last partition on the image. + +Also, the image MUST have the `parted` package installed for the +`partprobe` command. + +Step keys: + +* `resize-rootfs` — REQUIRED; value MUST be the tag for the root + filesystem. + +This is based on reading the changes by Peter Lawler to the +[image-specs](https://salsa.debian.org/raspi-team/image-specs.git) +repository to do the same thing. + +Example: + +~~~yaml +- resize-rootfs: root +~~~ diff -Nru vmdb2-0.19/vmdb/plugins/resize_plugin.py vmdb2-0.22/vmdb/plugins/resize_plugin.py --- vmdb2-0.19/vmdb/plugins/resize_plugin.py 1970-01-01 00:00:00.000000000 +0000 +++ vmdb2-0.22/vmdb/plugins/resize_plugin.py 2021-01-16 15:14:04.000000000 +0000 @@ -0,0 +1,116 @@ +# Copyright 2020 Lars Wirzenius, Peter Lawler +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import os + +import vmdb + + +SCRIPT = """\ +#!/bin/sh + +rootpart="$(findmnt -n -o SOURCE /)" +rootdev="/dev/$(lsblk -no pkname "$rootpart")" +partno="$(lsblk -n --list "$rootpart" | wc -l)" + +flock $rootdev sfdisk -f $rootdev -N "$partno" < +.\" +.\" This program is free software: you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation, either version 3 of the License, or +.\" (at your option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with this program. If not, see . +.\" +.TH VMDB2 1 +.SH NAME +vmdb2 \- create a disk image with Debian installed +.SH SYNOPSIS +.nh +.B vmdb2 +.RB [ \-\-image\fR=\fIFILE ] +.RB [ \-\-output\fR=\fIFILE ] +.RB [ \-\-rootfs\-tarball\fR=\fIFILE ] +.RB [ \-h | \-\-help ] +.RB [ \-\-version ] +.RB [ \-v | \-\-verbose ] +.RB [ \-\-log\fR=\fIFILE ] +.RI [ FILE ]... "" +.hy +.B vmdb +.IR SPECFILE +.SH DESCRIPTION +.B vmdb2 +reads a +.I "specification file" +that describes a disk image, +and builds the corresponding image. +The specification file uses YAML syntax, +and contains a list of steps. +Each step tells vmdb2 what to do next. +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +show this help message and exit +.TP +.BR \-\-image =\fIFILE +use existing image file/device +.I FILE +.TP +.BR \-\-output =\fIFILE +create new image +.I FILE +(the size comes from the specification file) +.TP +.BR \-\-rootfs\-tarball =\fIFILE +store rootfs cache tar archives in +.I FILE +.TP +.BR \-v ", " \-\-verbose +verbose output +.TP +.BR \-\-log =\fIFILE +write log entries to +.I FILE +(default is to not write log files at all) +.TP +.BR \-\-version +show program's version number and exit +.SH EXAMPLE +To build a disk image: +.PP +.nf +.RS +vmdb2 foo.vmdb --output foo.img -v --rootfs-tarball foo.tar.gz +.RE +.fi +.SH "SEE ALSO" +.BR debootstrap (8). +.PP +.I https://vmdb2.liw.fi/ +.PP +.IR /usr/share/doc/vmdb2/manual.html diff -Nru vmdb2-0.19/vmdb2.1.in vmdb2-0.22/vmdb2.1.in --- vmdb2-0.19/vmdb2.1.in 2019-02-15 04:50:32.000000000 +0000 +++ vmdb2-0.22/vmdb2.1.in 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -.\" Copyright 2017 Lars Wirzenius -.\" -.\" This program is free software: you can redistribute it and/or modify -.\" it under the terms of the GNU General Public License as published by -.\" the Free Software Foundation, either version 3 of the License, or -.\" (at your option) any later version. -.\" -.\" This program is distributed in the hope that it will be useful, -.\" but WITHOUT ANY WARRANTY; without even the implied warranty of -.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -.\" GNU General Public License for more details. -.\" -.\" You should have received a copy of the GNU General Public License -.\" along with this program. If not, see . -.\" -.TH VMDB2 1 -.SH NAME -vmdb2 \- create a disk image with Debian installed -.SH SYNOPSIS -.B vmdb -.RI [ options ] -.IR SPECFILE ... -.SH DESCRIPTION -.B vmdb2 -reads a -.I "specification file" -that describes a disk image, -and builds the corresponding image. -The specification file uses YAML syntax, -and contains a list of steps. -Each step tells vmdb2 what to do next. -.SH OPTIONS -.SH BUGS -This is unfinished at this time. -.SH EXAMPLE -To build a disk image: -.PP -.nf -.RS -vmdb2 foo.vmdb -.RE -.fi -.SH "SEE ALSO" -.BR vmdebootstrap (1). -.PP -See the -.B vmdb2 -manual and test suite for a full explanation of specification files. -This may be installed as -.IR /usr/share/doc/vmdb2/manual.html . \ No newline at end of file diff -Nru vmdb2-0.19/vmdb2.md vmdb2-0.22/vmdb2.md --- vmdb2-0.19/vmdb2.md 2020-08-12 05:53:37.000000000 +0000 +++ vmdb2-0.22/vmdb2.md 2021-01-16 15:14:04.000000000 +0000 @@ -334,6 +334,7 @@ author: Lars Wirzenius bindings: vmdb2.yaml functions: vmdb2.py +template: python documentclass: report ... diff -Nru vmdb2-0.19/without-tests vmdb2-0.22/without-tests --- vmdb2-0.19/without-tests 2020-08-12 05:53:37.000000000 +0000 +++ vmdb2-0.22/without-tests 2021-01-16 15:14:04.000000000 +0000 @@ -22,6 +22,7 @@ vmdb/plugins/mkpart_plugin.py vmdb/plugins/mount_plugin.py vmdb/plugins/qemudebootstrap_plugin.py +vmdb/plugins/resize_plugin.py vmdb/plugins/shell_plugin.py vmdb/plugins/unpack_rootfs_plugin.py vmdb/plugins/vgcreate_plugin.py diff -Nru vmdb2-0.19/yarns/100-mvp.yarn vmdb2-0.22/yarns/100-mvp.yarn --- vmdb2-0.19/yarns/100-mvp.yarn 2019-12-27 14:24:57.000000000 +0000 +++ vmdb2-0.22/yarns/100-mvp.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -114,7 +114,7 @@ ... { echo: "bar", teardown: "bar_teardown" } ... ] ... } - WHEN user runs vmdb2 -v happy.vmdb + WHEN user runs vmdb2 -v happy.vmdb --output happy.img THEN exit code is 0 AND stdout contains "foo" followed by "bar" AND stdout contains "bar" followed by "bar_teardown" @@ -159,7 +159,7 @@ ... { echo: "bar_step", teardown: "bar_teardown" } ... ] ... } - WHEN user runs vmdb2 -v unhappy.vmdb + WHEN user runs vmdb2 -v unhappy.vmdb --output=unhappy.img THEN exit code is 1 AND stdout contains "foo" followed by "yikes" AND stdout contains "yikes" followed by "WAT?!" diff -Nru vmdb2-0.19/yarns/900-implements.yarn vmdb2-0.22/yarns/900-implements.yarn --- vmdb2-0.19/yarns/900-implements.yarn 2019-12-27 14:24:57.000000000 +0000 +++ vmdb2-0.22/yarns/900-implements.yarn 2021-01-16 15:14:04.000000000 +0000 @@ -11,8 +11,13 @@ IMPLEMENTS WHEN user runs vmdb2 (.*) args = get_next_match() vmdb2 = os.path.join(srcdir, 'vmdb2') - exit, out, err = cliapp.runcmd_unchecked([vmdb2] + args.split()) - vars['exit'] = exit + p = subprocess.Popen( + [vmdb2] + args.split(), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + vars['exit'] = p.returncode vars['stdout'] = out.decode() vars['stderr'] = err.decode() diff -Nru vmdb2-0.19/yarns/lib.py vmdb2-0.22/yarns/lib.py --- vmdb2-0.19/yarns/lib.py 2019-02-15 04:50:32.000000000 +0000 +++ vmdb2-0.22/yarns/lib.py 2021-01-16 15:14:04.000000000 +0000 @@ -1,10 +1,10 @@ import os +import subprocess import sys -import cliapp from yarnutils import * -srcdir = os.environ['SRCDIR'] -datadir = os.environ['DATADIR'] +srcdir = os.environ["SRCDIR"] +datadir = os.environ["DATADIR"] vars = Variables(datadir)