diff -Nru debuerreotype-0.10/build-all.sh debuerreotype-0.14/build-all.sh --- debuerreotype-0.10/build-all.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/build-all.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -suites=( - unstable - testing - stable - oldstable - oldoldstable - - # just in case (will no-op with "not supported on 'arch'" unless it exists) - oldoldoldstable -) - -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" -source "$thisDir/scripts/.constants.sh" \ - --flags 'no-build' \ - -- \ - '[--no-build] ' \ - 'output 2017-05-08T00:00:00Z' - -eval "$dgetopt" -build=1 -while true; do - flag="$1"; shift - dgetopt-case "$flag" - case "$flag" in - --no-build) build= ;; # for skipping "docker build" - --) break ;; - *) eusage "unknown flag '$flag'" ;; - esac -done - -outputDir="${1:-}"; shift || eusage 'missing output-dir' -timestamp="${1:-}"; shift || eusage 'missing timestamp' - -mkdir -p "$outputDir" -outputDir="$(readlink -f "$outputDir")" - -ver="$("$thisDir/scripts/debuerreotype-version")" -ver="${ver%% *}" -dockerImage="debuerreotype/debuerreotype:$ver" -[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" - -mirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp")" -secmirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp" 'debian-security')" - -dpkgArch="$(docker run --rm "$dockerImage" dpkg --print-architecture | awk -F- '{ print $NF }')" -echo -echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --" -echo - -for suite in "${suites[@]}"; do - doSkip= - case "$suite" in - testing|unstable) ;; - *) - if ! wget --quiet --spider "$secmirror/dists/$suite/updates/main/binary-$dpkgArch/Packages.gz"; then - doSkip=1 - fi - ;; - esac - if ! wget --quiet --spider "$mirror/dists/$suite/main/binary-$dpkgArch/Packages.gz"; then - doSkip=1 - fi - if [ -n "$doSkip" ]; then - echo >&2 - echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping" - echo >&2 - continue - fi - "$thisDir/build.sh" --no-build --codename-copy "$outputDir" "$suite" "$timestamp" -done diff -Nru debuerreotype-0.10/build.sh debuerreotype-0.14/build.sh --- debuerreotype-0.10/build.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/build.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,339 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" -source "$thisDir/scripts/.constants.sh" \ - --flags 'no-build,codename-copy' \ - --flags 'eol,arch:,qemu' \ - -- \ - '[--no-build] [--codename-copy] [--eol] [--arch=] [--qemu] ' \ - 'output stretch 2017-05-08T00:00:00Z ---codename-copy output stable 2017-05-08T00:00:00Z ---eol output squeeze 2016-03-14T00:00:00Z ---eol --arch i386 output sarge 2016-03-14T00:00:00Z' \ - -eval "$dgetopt" -build=1 -codenameCopy= -eol= -arch= -qemu= -while true; do - flag="$1"; shift - dgetopt-case "$flag" - case "$flag" in - --no-build) build= ;; # for skipping "docker build" - --codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work) - --eol) eol=1 ;; # for using "archive.debian.org" - --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init - --qemu) qemu=1 ;; # for using "qemu-debootstrap" - --) break ;; - *) eusage "unknown flag '$flag'" ;; - esac -done - -outputDir="${1:-}"; shift || eusage 'missing output-dir' -suite="${1:-}"; shift || eusage 'missing suite' -timestamp="${1:-}"; shift || eusage 'missing timestamp' - -mkdir -p "$outputDir" -outputDir="$(readlink -f "$outputDir")" - -securityArgs=( - --cap-add SYS_ADMIN - --cap-drop SETFCAP -) -if docker info | grep -q apparmor; then - # AppArmor blocks mount :) - securityArgs+=( - --security-opt apparmor=unconfined - ) -fi - -if [ "$suite" = 'potato' ]; then - # --debian-eol potato wants to run "chroot ... mount ... /proc" which gets blocked (i386, ancient binaries, blah blah blah) - securityArgs+=( - --security-opt seccomp=unconfined - ) -fi - -ver="$("$thisDir/scripts/debuerreotype-version")" -ver="${ver%% *}" -dockerImage="debuerreotype/debuerreotype:$ver" -[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" -if [ -n "$qemu" ]; then - [ -z "$build" ] || docker build -t "$dockerImage-qemu" - <<-EODF - FROM $dockerImage - RUN apt-get update && apt-get install -y --no-install-recommends qemu-user-static && rm -rf /var/lib/apt/lists/* - EODF - dockerImage="$dockerImage-qemu" -fi - -docker run \ - --rm \ - "${securityArgs[@]}" \ - --tmpfs /tmp:dev,exec,suid,noatime \ - -w /tmp \ - -e suite="$suite" \ - -e timestamp="$timestamp" \ - -e codenameCopy="$codenameCopy" \ - -e eol="$eol" -e arch="$arch" -e qemu="$qemu" \ - -e TZ='UTC' -e LC_ALL='C' \ - --hostname debuerreotype \ - "$dockerImage" \ - bash -Eeuo pipefail -c ' - set -x - - epoch="$(date --date "$timestamp" +%s)" - serial="$(date --date "@$epoch" +%Y%m%d)" - dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- "{ print \$NF }")}" - - exportDir="output" - outputDir="$exportDir/$serial/$dpkgArch/$suite" - - touch_epoch() { - while [ "$#" -gt 0 ]; do - local f="$1"; shift - touch --no-dereference --date="@$epoch" "$f" - done - } - - debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" - - for archive in "" security; do - if [ -z "$eol" ]; then - snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "${archive:+debian-${archive}}")" - else - snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "debian-archive")/debian${archive:+-${archive}}" - fi - snapshotUrlFile="$exportDir/$serial/$dpkgArch/snapshot-url${archive:+-${archive}}" - mkdir -p "$(dirname "$snapshotUrlFile")" - echo "$snapshotUrl" > "$snapshotUrlFile" - touch_epoch "$snapshotUrlFile" - done - - export GNUPGHOME="$(mktemp -d)" - keyring="$GNUPGHOME/debian-archive-$suite-keyring.gpg" - if [ "$suite" = potato ]; then - # src:debian-archive-keyring was created in 2006, thus does not include a key for potato - gpg --batch --no-default-keyring --keyring "$keyring" \ - --keyserver ha.pool.sks-keyservers.net \ - --recv-keys 8FD47FF1AA9372C37043DC28AA7DEB7B722F1AED - else - # check against all releases (ie, combine both "debian-archive-keyring.gpg" and "debian-archive-removed-keys.gpg"), since we cannot really know whether the target release became EOL later than the snapshot date we are targeting - gpg --batch --no-default-keyring --keyring "$keyring" --import \ - /usr/share/keyrings/debian-archive-keyring.gpg \ - /usr/share/keyrings/debian-archive-removed-keys.gpg - fi - - snapshotUrl="$(< "$exportDir/$serial/$dpkgArch/snapshot-url")" - mkdir -p "$outputDir" - wget -O "$outputDir/Release.gpg" "$snapshotUrl/dists/$suite/Release.gpg" - wget -O "$outputDir/Release" "$snapshotUrl/dists/$suite/Release" - gpgv \ - --keyring "$keyring" \ - "$outputDir/Release.gpg" \ - "$outputDir/Release" - - codename="$(awk -F ": " "\$1 == \"Codename\" { print \$2; exit }" "$outputDir/Release")" - if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then - # if codename already is the same as suite, then making a copy does not make any sense - codenameCopy= - fi - if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then - echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite" - exit 1 - fi - - { - initArgs=( --arch="$dpkgArch" ) - if [ -z "$eol" ]; then - initArgs+=( --debian ) - else - initArgs+=( --debian-eol ) - fi - initArgs+=( --keyring "$keyring" ) - - # disable merged-usr (for now?) due to the following compelling arguments: - # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) - # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) - # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 - initArgs+=( --no-merged-usr ) - - if [ -n "$qemu" ]; then - initArgs+=( --debootstrap="qemu-debootstrap" ) - fi - - debuerreotype-init "${initArgs[@]}" rootfs "$suite" "@$epoch" - - if [ -n "$eol" ]; then - debuerreotype-gpgv-ignore-expiration-config rootfs - fi - - debuerreotype-minimizing-config rootfs - debuerreotype-apt-get rootfs update -qq - debuerreotype-apt-get rootfs dist-upgrade -yqq - - aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" rootfs)" - if dpkg --compare-versions "$aptVersion" ">=" "0.7.14~"; then - # https://salsa.debian.org/apt-team/apt/commit/06d79436542ccf3e9664306da05ba4c34fba4882 - noInstallRecommends="--no-install-recommends" - else - # --debian-eol etch and lower do not support --no-install-recommends - noInstallRecommends="-o APT::Install-Recommends=0" - fi - - if [ -n "$eol" ] && dpkg --compare-versions "$aptVersion" ">=" "0.7.26~"; then - # https://salsa.debian.org/apt-team/apt/commit/1ddb859611d2e0f3d9ea12085001810f689e8c99 - echo "Acquire::Check-Valid-Until \"false\";" > rootfs/etc/apt/apt.conf.d/check-valid-until.conf - # TODO make this a real script so it can have a nice comment explaining why we do it for EOL releases? - fi - - # make a couple copies of rootfs so we can create other variants - for variant in slim sbuild; do - mkdir "rootfs-$variant" - tar -cC rootfs . | tar -xC "rootfs-$variant" - done - - # prefer iproute2 if it exists - iproute=iproute2 - if ! debuerreotype-apt-get rootfs install -qq -s iproute2 &> /dev/null; then - # poor wheezy - iproute=iproute - fi - ping=iputils-ping - if debuerreotype-chroot rootfs bash -c "command -v ping > /dev/null"; then - # if we already have "ping" (as in --debian-eol potato), skip installing any extra ping package - ping= - fi - debuerreotype-apt-get rootfs install -y $noInstallRecommends $ping $iproute - - debuerreotype-slimify rootfs-slim - - # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild - # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 - # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 - debuerreotype-apt-get rootfs-sbuild install -y $noInstallRecommends build-essential fakeroot - - create_artifacts() { - local targetBase="$1"; shift - local rootfs="$1"; shift - local suite="$1"; shift - local variant="$1"; shift - - # make a copy of the snapshot-facing sources.list file before we overwrite it - cp "$rootfs/etc/apt/sources.list" "$targetBase.sources-list-snapshot" - touch_epoch "$targetBase.sources-list-snapshot" - - local tarArgs=() - if [ -n "$qemu" ]; then - tarArgs+=( --exclude="./usr/bin/qemu-*-static" ) - fi - - if [ "$variant" != "sbuild" ]; then - debuerreotype-debian-sources-list $([ -z "$eol" ] || echo "--eol") "$rootfs" "$suite" - else - # sbuild needs "deb-src" entries - debuerreotype-debian-sources-list --deb-src $([ -z "$eol" ] || echo "--eol") "$rootfs" "$suite" - - # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes - # (which are used in sbuild for "--extra-package") - # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - # ... - # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" - # TODO figure out the bug and fix it in APT instead /o\ - - # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") - # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 - tarArgs+=( --include-dev ) - fi - - case "$suite" in - sarge) - # for some reason, sarge creates "/var/cache/man/index.db" with some obvious embedded unix timestamps (but if we exclude it, "man" still works properly, so *shrug*) - tarArgs+=( --exclude ./var/cache/man/index.db ) - ;; - - woody) - # woody not only contains "exim", but launches it during our build process and tries to email "root@debuerreotype" (which fails and creates non-reproducibility) - tarArgs+=( --exclude ./var/spool/exim --exclude ./var/log/exim ) - ;; - - potato) - tarArgs+=( - # for some reason, pototo leaves a core dump (TODO figure out why??) - --exclude "./core" - --exclude "./qemu*.core" - # also, it leaves some junk in /tmp (/tmp/fdmount.conf.tmp.XXX) - --exclude "./tmp/fdmount.conf.tmp.*" - ) - ;; - esac - - debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" - du -hsx "$targetBase.tar.xz" - - sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" - touch_epoch "$targetBase.tar.xz.sha256" - - debuerreotype-chroot "$rootfs" bash -c " - if ! dpkg-query -W 2> /dev/null; then - # --debian-eol woody has no dpkg-query - dpkg -l - fi - " > "$targetBase.manifest" - echo "$epoch" > "$targetBase.debuerreotype-epoch" - debuerreotype-version > "$targetBase.debuerreotype-version" - touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" "$targetBase.debuerreotype-version" - - for f in debian_version os-release apt/sources.list; do - targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" - if [ -e "$rootfs/etc/$f" ]; then - # /etc/os-release does not exist in --debian-eol squeeze, for example (hence the existence check) - cp "$rootfs/etc/$f" "$targetFile" - touch_epoch "$targetFile" - fi - done - } - - for rootfs in rootfs*/; do - rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... - - du -hsx "$rootfs" - - variant="${rootfs#rootfs}" # "", "-slim", ... - variant="${variant#-}" # "", "slim", ... - - variantDir="$outputDir/$variant" - mkdir -p "$variantDir" - - targetBase="$variantDir/rootfs" - - create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" - done - - if [ -n "$codenameCopy" ]; then - codenameDir="$exportDir/$serial/$dpkgArch/$codename" - mkdir -p "$codenameDir" - tar -cC "$outputDir" --exclude="**/rootfs.*" . | tar -xC "$codenameDir" - - for rootfs in rootfs*/; do - rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... - - variant="${rootfs#rootfs}" # "", "-slim", ... - variant="${variant#-}" # "", "slim", ... - - variantDir="$codenameDir/$variant" - targetBase="$variantDir/rootfs" - - # point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite) - debuerreotype-debian-sources-list --snapshot $([ -z "$eol" ] || echo "--eol") "$rootfs" "$codename" - - create_artifacts "$targetBase" "$rootfs" "$codename" "$variant" - done - fi - } >&2 - - tar -cC "$exportDir" . - ' | tar -xvC "$outputDir" diff -Nru debuerreotype-0.10/debian/changelog debuerreotype-0.14/debian/changelog --- debuerreotype-0.10/debian/changelog 2021-03-26 22:36:13.000000000 +0000 +++ debuerreotype-0.14/debian/changelog 2022-03-22 16:36:10.000000000 +0000 @@ -1,9 +1,22 @@ -debuerreotype (0.10-2ubuntu1) hirsute; urgency=medium +debuerreotype (0.14-1) unstable; urgency=medium - * Depend on debian-archive-keyring for autopkgtests to succeed - (Closes: #985963) + [ Tianon Gravi ] + * Add several more architecture hashes to autopkgtests: + - armel + - armhf + - i386 + - mips64el + - ppc64el + - s390x - -- Gianfranco Costamagna Fri, 26 Mar 2021 23:36:13 +0100 + [ Gianfranco Costamagna ] + * Depend on debian-archive-keyring for autopkgtests (Closes: #985963) + + [ Tianon Gravi ] + * Update to 0.14 upstream release + - most notable change is more correct "/etc/machine-id" handling + + -- Tianon Gravi Tue, 22 Mar 2022 09:36:10 -0700 debuerreotype (0.10-2) unstable; urgency=medium diff -Nru debuerreotype-0.10/debian/tests/control debuerreotype-0.14/debian/tests/control --- debuerreotype-0.10/debian/tests/control 2021-03-26 22:35:49.000000000 +0000 +++ debuerreotype-0.14/debian/tests/control 2021-03-26 22:47:05.000000000 +0000 @@ -8,4 +8,3 @@ wget, xz-utils Restrictions: allow-stderr, needs-root -#Architecture: amd64, arm64 diff -Nru debuerreotype-0.10/debian/tests/stretch debuerreotype-0.14/debian/tests/stretch --- debuerreotype-0.10/debian/tests/stretch 2021-03-26 22:35:22.000000000 +0000 +++ debuerreotype-0.14/debian/tests/stretch 2022-03-22 16:27:42.000000000 +0000 @@ -6,13 +6,22 @@ expectedEpoch='1483228800' +# https://people.debian.org/~tianon/debuerreotype/ declare -A expectedSha256s=( - ['amd64']='26490ed3400a5029b8b5939c6cebd38691e28ea5c616bb54f25f758125417c5f' - ['arm64']='45e5e0c6da27db14de19a600663140db1f85f7516a856bf00957436e93e1f684' + ['amd64']='cff511450d3ddec4defc51be1850ca44e527e126aabfa1b6649354f2f2a568f4' + ['arm64']='81ad4f5e7c10b15fc8389c1386e1fe1b2d05bc61d404e10473f5f5a0acde464c' + ['armel']='596117e125f6e0828471564bd5d4f9bdd1ed654f7d84c21e7ab9d640e789763f' + ['armhf']='9e0f63a3d5442fcd1408d61a9c592355138334ddc5c3fb471d5f131511cf286c' + ['i386']='b4e574904703114291f1c42bd80744b1da333d95cf5c00e8e71ba09241ca66ae' + ['mips64el']='767982e9ca353058d9ca44f410e0367071ebf9938a4a996e3ed9139d57bc987f' + ['ppc64el']='99bc8a072448360be9ff741b98466835bc0f270041784cacbcb3803f8d3fbb6a' + ['s390x']='7ad6fa1abaf9d8cb3fd9d1257e24363bc842f0fe204e98d0a2aa995541b62b1c' ) dpkgArch="$(dpkg --print-architecture)" expectedSha256="${expectedSha256s["$dpkgArch"]:-}" +expectedCompareUrl="https://people.debian.org/~tianon/debuerreotype/$suite--$timestamp--0.14--$dpkgArch--$expectedSha256.txz" + tempDir="$(mktemp -d)" trap "rm -rf '$tempDir'" EXIT rootfs="$tempDir/rootfs" @@ -55,11 +64,11 @@ set +x echo >&2 echo >&2 'ERROR: expected SHA256 does not match actual -- downloading pristine source to compare (via diffoscope)' + echo >&2 " - $expectedCompareUrl" echo >&2 ) - toCompare="https://people.debian.org/~tianon/debuerreotype/$suite--$timestamp--$dpkgArch--$expectedSha256.txz" - wget -qO "$tempDir/expected.txz" "$toCompare" + wget -qO "$tempDir/expected.txz" "$expectedCompareUrl" xz -d < "$tempDir/expected.txz" > "$tempDir/expected.tar" diffoscope >&2 "$tempDir/expected.tar" "$tempDir/actual.tar" exit 1 diff -Nru debuerreotype-0.10/Dockerfile debuerreotype-0.14/Dockerfile --- debuerreotype-0.10/Dockerfile 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/Dockerfile 2022-03-18 19:37:10.000000000 +0000 @@ -1,27 +1,58 @@ # docker run --cap-add SYS_ADMIN --cap-drop SETFCAP --tmpfs /tmp:dev,exec,suid,noatime ... # bootstrapping a new architecture? -# ./scripts/debuerreotype-init /tmp/docker-rootfs stretch now +# ./scripts/debuerreotype-init /tmp/docker-rootfs bullseye now # ./scripts/debuerreotype-minimizing-config /tmp/docker-rootfs -# ./scripts/debuerreotype-debian-sources-list /tmp/docker-rootfs stretch -# ./scripts/debuerreotype-tar /tmp/docker-rootfs - | docker import - debian:stretch-slim +# ./scripts/debuerreotype-debian-sources-list /tmp/docker-rootfs bullseye +# ./scripts/debuerreotype-tar /tmp/docker-rootfs - | docker import - debian:bullseye-slim # alternate: -# debootstrap --variant=minbase stretch /tmp/docker-rootfs -# tar -cC /tmp/docker-rootfs . | docker import - debian:stretch-slim +# debootstrap --variant=minbase bullseye /tmp/docker-rootfs +# tar -cC /tmp/docker-rootfs . | docker import - debian:bullseye-slim # (or your own favorite set of "debootstrap" commands to create a base image for building this one FROM) -FROM debian:stretch-slim +FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + debian-ports-archive-keyring \ debootstrap \ wget ca-certificates \ xz-utils \ \ gnupg dirmngr \ - && rm -rf /var/lib/apt/lists/* + ; \ + rm -rf /var/lib/apt/lists/* + +# fight the tyrrany of HSTS (which destroys our ability to transparently cache snapshot.debian.org responses) +ENV WGETRC /.wgetrc +RUN echo 'hsts=0' >> "$WGETRC" + +# https://github.com/debuerreotype/debuerreotype/issues/100 +# https://tracker.debian.org/pkg/distro-info-data +# http://snapshot.debian.org/package/distro-info-data/ +# http://snapshot.debian.org/package/distro-info-data/0.51/ +RUN set -eux; \ + wget -O distro-info-data.deb 'http://snapshot.debian.org/archive/debian/20210724T033023Z/pool/main/d/distro-info-data/distro-info-data_0.51_all.deb'; \ + echo 'c5f4a3bd999d3d79612dfec285e4afc4f6248648 *distro-info-data.deb' | sha1sum --strict --check -; \ + apt-get install -y ./distro-info-data.deb; \ + rm distro-info-data.deb; \ + [ -s /usr/share/distro-info/debian.csv ] + +# https://bugs.debian.org/973852 +# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/63 +# https://people.debian.org/~tianon/debootstrap-mr-63--download_main.patch +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends patch; \ + rm -rf /var/lib/apt/lists/*; \ + wget -O debootstrap-download-main.patch 'https://people.debian.org/~tianon/debootstrap-mr-63--download_main.patch'; \ + echo 'ceae8f508a9b49236fa4519a44a584e6c774aa0e4446eb1551f3b69874a4cde5 *debootstrap-download-main.patch' | sha256sum --strict --check -; \ + patch --input=debootstrap-download-main.patch /usr/share/debootstrap/functions; \ + rm debootstrap-download-main.patch # see ".dockerignore" COPY . /opt/debuerreotype -RUN set -ex; \ +RUN set -eux; \ cd /opt/debuerreotype/scripts; \ for f in debuerreotype-*; do \ ln -svL "$PWD/$f" "/usr/local/bin/$f"; \ @@ -34,12 +65,14 @@ # a few example md5sum values for amd64: -# debuerreotype-init test-stretch stretch 2017-05-08T00:00:00Z +# debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg test-stretch stretch 2017-05-08T00:00:00Z # debuerreotype-tar test-stretch test-stretch.tar # md5sum test-stretch.tar -# 14206d5b9b2991e98f5214c3d310e4fa +# 694f02c53651673ebe094cae3bcbb06d +# ./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs stretch 2017-05-08T00:00:00Z; debuerreotype-tar /tmp/rootfs - | md5sum' -# debuerreotype-init test-jessie jessie 2017-05-08T00:00:00Z +# debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg test-jessie jessie 2017-05-08T00:00:00Z # debuerreotype-tar test-jessie test-jessie.tar # md5sum test-jessie.tar -# 57f98d3636000630080e5ba208508e10 +# 354cedd99c08d213d3493a7cf0aaaad6 +# ./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs jessie 2017-05-08T00:00:00Z; debuerreotype-tar /tmp/rootfs - | md5sum' diff -Nru debuerreotype-0.10/.dockerignore debuerreotype-0.14/.dockerignore --- debuerreotype-0.10/.dockerignore 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/.dockerignore 2022-03-18 19:37:10.000000000 +0000 @@ -1,4 +1,5 @@ ** !VERSION +!examples/ !scripts/ diff -Nru debuerreotype-0.10/.docker-image.sh debuerreotype-0.14/.docker-image.sh --- debuerreotype-0.10/.docker-image.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/.docker-image.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +thisDir="$(readlink -vf "$BASH_SOURCE")" +thisDir="$(dirname "$thisDir")" + +ver="$("$thisDir/scripts/debuerreotype-version")" +ver="${ver%% *}" +dockerImage="debuerreotype/debuerreotype:$ver" + +echo "$dockerImage" diff -Nru debuerreotype-0.10/docker-run.sh debuerreotype-0.14/docker-run.sh --- debuerreotype-0.10/docker-run.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/docker-run.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# usage: mkdir -p output && ./run-script.sh ./examples/debian.sh output ... + +thisDir="$(readlink -vf "$BASH_SOURCE")" +thisDir="$(dirname "$thisDir")" + +source "$thisDir/scripts/.constants.sh" \ + --flags 'image:' \ + --flags 'no-bind' \ + --flags 'no-build' \ + --flags 'pull' \ + -- \ + '[--image=foo/bar:baz] [--no-build] [--no-bind] [--pull] [script/command]' \ + './examples/debian.sh output stretch 2017-05-08T00:00:00Z +--no-build --image=debuerreotype:ubuntu ./examples/ubuntu.sh output xenial' + +eval "$dgetopt" +image= +build=1 +bindMount=1 +pull= +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --image) image="$1"; shift ;; + --no-bind) bindMount= ;; + --no-build) build= ;; + --pull) pull=1 ;; + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +if [ -z "$image" ]; then + image="$("$thisDir/.docker-image.sh")" +fi +if [ -n "$build" ]; then + docker build ${pull:+--pull} --tag "$image" "$thisDir" +elif [ -n "$pull" ]; then + docker pull "$image" +else + # make sure "docker run" doesn't pull (we have `--no-build` and no explicit `--pull`) + docker image inspect "$image" > /dev/null +fi + +args=( + --hostname debuerreotype + --init + --interactive + --rm + + # we ought to be able to mount/unshare + --cap-add SYS_ADMIN + # make sure we don't get extended attributes + --cap-drop SETFCAP + + # AppArmor also blocks mount/unshare :) + --security-opt apparmor=unconfined + + # --debian-eol potato wants to run "chroot ... mount ... /proc" which gets blocked (i386, ancient binaries, blah blah blah) + --security-opt seccomp=unconfined + # (other arches see this occasionally too) + + --tmpfs /tmp:dev,exec,suid,noatime + --env TMPDIR=/tmp + + --workdir /workdir +) +if [ -n "$bindMount" ]; then + args+=( --mount "type=bind,src=$PWD,dst=/workdir" ) +else + args+=( --volume /workdir ) +fi + +if [ -t 0 ] && [ -t 1 ]; then + args+=( --tty ) +fi + +exec docker run "${args[@]}" "$image" "$@" diff -Nru debuerreotype-0.10/examples/debian-all.sh debuerreotype-0.14/examples/debian-all.sh --- debuerreotype-0.10/examples/debian-all.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/examples/debian-all.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +suites=( + unstable + testing + stable + oldstable + oldoldstable + + # just in case (will no-op with "not supported on 'arch'" unless it exists) + oldoldoldstable +) + +debuerreotypeScriptsDir="$(which debuerreotype-init)" +debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" +debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" + +source "$debuerreotypeScriptsDir/.constants.sh" \ + --flags 'arch:' \ + -- \ + '[--arch=] ' \ + 'output 2017-05-08T00:00:00Z' + +eval "$dgetopt" +arch= +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --arch) arch="$1"; shift ;; + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +outputDir="${1:-}"; shift || eusage 'missing output-dir' +timestamp="${1:-}"; shift || eusage 'missing timestamp' + +debianArgs=( --codename-copy ) + +mirror="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "$timestamp")" +secmirror="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "$timestamp" 'debian-security')" + +dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" +echo +echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --" +echo +debianArgs+=( --arch="$dpkgArch" ) + +thisDir="$(readlink -vf "$BASH_SOURCE")" +thisDir="$(dirname "$thisDir")" + +_eol-date() { + local codename="$1"; shift # "bullseye", "buster", etc. + if [ ! -s /usr/share/distro-info/debian.csv ]; then + echo >&2 "warning: looks like we are missing 'distro-info-data' (/usr/share/distro-info/debian.csv); cannot calculate EOL dates accurately!" + exit 1 + fi + awk -F, -v codename="$codename" ' + NR == 1 { + headers = NF + for (i = 1; i <= headers; i++) { + header[i] = $i + } + next + } + { + delete row + for (i = 1; i <= NF && i <= headers; i++) { + row[header[i]] = $i + } + } + row["series"] == codename { + if (row["eol-lts"] != "") { + eol = row["eol-lts"] + exit 0 + } + if (row["eol"] != "") { + eol = row["eol"] + exit 0 + } + exit 1 + } + END { + if (eol != "") { + print eol + exit 0 + } + exit 1 + } + ' /usr/share/distro-info/debian.csv +} + +_codename() { + local dist="$1"; shift + + local release + if release="$(wget --quiet --output-document=- "$mirror/dists/$dist/InRelease")"; then + : + elif release="$(wget --quiet --output-document=- "$mirror/dists/$dist/Release")"; then + : + else + return 1 + fi + + local codename + codename="$(awk '$1 == "Codename:" { print $2 }' <<<"$release")" + [ -n "$codename" ] || return 1 + echo "$codename" +} + +_check() { + local host="$1"; shift # "$mirror", "$secmirror" + local dist="$1"; shift # "$suite-security", "$suite/updates", "$suite" + local comp="${1:-main}" + + if wget --quiet --spider "$host/dists/$dist/$comp/binary-$dpkgArch/Packages.xz"; then + return 0 + fi + + if wget --quiet --spider "$host/dists/$dist/$comp/binary-$dpkgArch/Packages.gz"; then + return 0 + fi + + return 1 +} + +for suite in "${suites[@]}"; do + doSkip= + case "$suite" in + testing | unstable) ;; + + *) + # https://lists.debian.org/debian-devel-announce/2019/07/msg00004.html + if \ + ! _check "$secmirror" "$suite-security" \ + && ! _check "$secmirror" "$suite/updates" \ + ; then + doSkip=1 + fi + if [ -z "$doSkip" ] && codename="$(_codename "$suite")" && eol="$(_eol-date "$codename")"; then + epoch="$(date --date "$timestamp" '+%s')" + eolEpoch="$(date --date "$eol" '+%s')" + if [ "$epoch" -ge "$eolEpoch" ]; then + echo >&2 + echo >&2 "warning: '$suite' ('$codename') is EOL at '$timestamp' ('$eol'); skipping" + echo >&2 + continue + fi + fi + ;; + esac + if ! _check "$mirror" "$suite"; then + doSkip=1 + fi + if [ -n "$doSkip" ]; then + echo >&2 + echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping" + echo >&2 + continue + fi + "$thisDir/debian.sh" "${debianArgs[@]}" "$outputDir" "$suite" "$timestamp" +done diff -Nru debuerreotype-0.10/examples/debian.sh debuerreotype-0.14/examples/debian.sh --- debuerreotype-0.10/examples/debian.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/examples/debian.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,363 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +debuerreotypeScriptsDir="$(which debuerreotype-init)" +debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" +debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" + +source "$debuerreotypeScriptsDir/.constants.sh" \ + --flags 'codename-copy' \ + --flags 'eol,ports' \ + --flags 'arch:' \ + --flags 'include:,exclude:' \ + -- \ + '[--codename-copy] [--eol] [--ports] [--arch=] ' \ + 'output stretch 2017-05-08T00:00:00Z +--codename-copy output stable 2017-05-08T00:00:00Z +--eol output squeeze 2016-03-14T00:00:00Z +--eol --arch i386 output sarge 2016-03-14T00:00:00Z' + +eval "$dgetopt" +codenameCopy= +eol= +ports= +include= +exclude= +arch= +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work) + --eol) eol=1 ;; # for using "archive.debian.org" + --ports) ports=1 ;; # for using "debian-ports" + --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init + --include) include="${include:+$include,}$1"; shift ;; + --exclude) exclude="${exclude:+$exclude,}$1"; shift ;; + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +outputDir="${1:-}"; shift || eusage 'missing output-dir' +suite="${1:-}"; shift || eusage 'missing suite' +timestamp="${1:-}"; shift || eusage 'missing timestamp' + +set -x + +outputDir="$(readlink -ve "$outputDir")" + +tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" +trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT + +export TZ='UTC' LC_ALL='C' + +epoch="$(date --date "$timestamp" +%s)" +serial="$(date --date "@$epoch" +%Y%m%d)" +dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" + +exportDir="$tmpDir/output" +archDir="$exportDir/$serial/$dpkgArch" +tmpOutputDir="$archDir/$suite" + +touch_epoch() { + while [ "$#" -gt 0 ]; do + local f="$1"; shift + touch --no-dereference --date="@$epoch" "$f" + done +} + +for archive in '' security; do + snapshotUrlFile="$archDir/snapshot-url${archive:+-${archive}}" + mirrorArgs=() + if [ -n "$ports" ]; then + mirrorArgs+=( --ports ) + fi + if [ -n "$eol" ]; then + mirrorArgs+=( --eol ) + fi + mirrorArgs+=( "@$epoch" "$suite${archive:+-$archive}" "$dpkgArch" main ) + if ! mirrors="$("$debuerreotypeScriptsDir/.debian-mirror.sh" "${mirrorArgs[@]}")"; then + if [ "$archive" = 'security' ]; then + # if we fail to find the security mirror, we're probably not security supported (which is ~fine) + continue + else + exit 1 + fi + fi + eval "$mirrors" + [ -n "$snapshotMirror" ] + snapshotUrlDir="$(dirname "$snapshotUrlFile")" + mkdir -p "$snapshotUrlDir" + echo "$snapshotMirror" > "$snapshotUrlFile" + touch_epoch "$snapshotUrlFile" +done + +initArgs=( + --arch "$dpkgArch" +) + +if [ -z "$eol" ]; then + initArgs+=( --debian ) +else + initArgs+=( --debian-eol ) +fi +if [ -n "$ports" ]; then + initArgs+=( + --debian-ports + --include=debian-ports-archive-keyring + ) +fi + +export GNUPGHOME="$tmpDir/gnupg" +mkdir -p "$GNUPGHOME" +keyring="$tmpDir/debian-archive-$suite-keyring.gpg" +if [ "$suite" = 'slink' ]; then + # slink (2.1) introduced apt, but without PGP 😅 + initArgs+=( --no-check-gpg ) +elif [ "$suite" = 'potato' ]; then + # src:debian-archive-keyring was created in 2006, thus does not include a key for potato (2.2; EOL in 2003) + gpg --batch --no-default-keyring --keyring "$keyring" \ + --keyserver keyserver.ubuntu.com \ + --recv-keys 8FD47FF1AA9372C37043DC28AA7DEB7B722F1AED + initArgs+=( --keyring "$keyring" ) +else + # check against all releases (ie, combine both "debian-archive-keyring.gpg" and "debian-archive-removed-keys.gpg"), since we cannot really know whether the target release became EOL later than the snapshot date we are targeting + gpg --batch --no-default-keyring --keyring "$keyring" --import \ + /usr/share/keyrings/debian-archive-keyring.gpg \ + /usr/share/keyrings/debian-archive-removed-keys.gpg + if [ -n "$ports" ]; then + gpg --batch --no-default-keyring --keyring "$keyring" --import \ + /usr/share/keyrings/debian-ports-archive-keyring.gpg \ + /usr/share/keyrings/debian-ports-archive-keyring-removed.gpg + fi + initArgs+=( --keyring "$keyring" ) +fi + +mkdir -p "$tmpOutputDir" + +mirror="$(< "$archDir/snapshot-url")" +if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then + gpgv \ + --keyring "$keyring" \ + --output "$tmpOutputDir/Release" \ + "$tmpOutputDir/InRelease" + [ -s "$tmpOutputDir/Release" ] +elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then + rm -f "$tmpOutputDir/InRelease" # remove wget leftovers + gpgv \ + --keyring "$keyring" \ + "$tmpOutputDir/Release.gpg" \ + "$tmpOutputDir/Release" + [ -s "$tmpOutputDir/Release" ] +elif [ "$suite" = 'slink' ]; then + # "Release" files were introduced in potato (2.2+) + rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers +else + echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" + exit 1 +fi +codename= +if [ -f "$tmpOutputDir/Release" ]; then + codename="$(awk -F ': ' '$1 == "Codename" { print $2; exit }' "$tmpOutputDir/Release")" +fi +if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then + # if codename already is the same as suite, then making a copy does not make any sense + codenameCopy= +fi +if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then + echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite" + exit 1 +fi + +initArgs+=( + # disable merged-usr (for now?) due to the following compelling arguments: + # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) + # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) + # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 + --no-merged-usr +) + +if [ -n "$include" ]; then + initArgs+=( --include="$include" ) +fi +if [ -n "$exclude" ]; then + initArgs+=( --exclude="$exclude" ) +fi + +rootfsDir="$tmpDir/rootfs" +debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "@$epoch" + +if [ -n "$eol" ]; then + debuerreotype-gpgv-ignore-expiration-config "$rootfsDir" +fi + +debuerreotype-minimizing-config "$rootfsDir" + +debuerreotype-apt-get "$rootfsDir" update -qq + +aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" +if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then + debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq +else + debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq +fi + +if dpkg --compare-versions "$aptVersion" '>=' '0.7.14~'; then + # https://salsa.debian.org/apt-team/apt/commit/06d79436542ccf3e9664306da05ba4c34fba4882 + noInstallRecommends='--no-install-recommends' +else + # etch (4.0) and lower do not support --no-install-recommends + noInstallRecommends='-o APT::Install-Recommends=0' +fi + +if [ -n "$eol" ] && dpkg --compare-versions "$aptVersion" '>=' '0.7.26~'; then + # https://salsa.debian.org/apt-team/apt/commit/1ddb859611d2e0f3d9ea12085001810f689e8c99 + echo 'Acquire::Check-Valid-Until "false";' > "$rootfsDir"/etc/apt/apt.conf.d/check-valid-until.conf + # TODO make this a real script so it can have a nice comment explaining why we do it for EOL releases? +fi + +# copy the rootfs to create other variants +mkdir "$rootfsDir"-slim +tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim + +# for historical reasons (related to their usefulness in debugging non-working container networking in container early days before "--network container:xxx"), Debian 10 and older non-slim images included both "ping" and "ip" above "minbase", but in 11+ (Bullseye), that will no longer be the case and we will instead be a faithful minbase again :D +epoch2021="$(date --date '2021-01-01 00:00:00' +%s)" +if [ "$epoch" -lt "$epoch2021" ] || { isDebianBusterOrOlder="$([ -f "$rootfsDir/etc/os-release" ] && source "$rootfsDir/etc/os-release" && [ -n "${VERSION_ID:-}" ] && [ "${VERSION_ID%%.*}" -le 10 ] && echo 1)" && [ -n "$isDebianBusterOrOlder" ]; }; then + # prefer iproute2 if it exists + iproute=iproute2 + if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then + # poor wheezy + iproute=iproute + fi + ping=iputils-ping + if debuerreotype-chroot "$rootfsDir" bash -c 'command -v ping > /dev/null'; then + # if we already have "ping" (as in potato, 2.2), skip installing any extra ping package + ping= + fi + debuerreotype-apt-get "$rootfsDir" install -y $noInstallRecommends $ping $iproute +fi + +debuerreotype-slimify "$rootfsDir"-slim + +sourcesListArgs=() +[ -z "$eol" ] || sourcesListArgs+=( --eol ) +[ -z "$ports" ] || sourcesListArgs+=( --ports ) + +create_artifacts() { + local targetBase="$1"; shift + local rootfs="$1"; shift + local suite="$1"; shift + local variant="$1"; shift + + # make a copy of the snapshot-facing sources.list file before we overwrite it + cp "$rootfs/etc/apt/sources.list" "$targetBase.sources-list-snapshot" + touch_epoch "$targetBase.sources-list-snapshot" + + debuerreotype-debian-sources-list "${sourcesListArgs[@]}" "$rootfs" "$suite" + + local tarArgs=( + # https://www.freedesktop.org/software/systemd/man/machine-id.html + --exclude ./etc/machine-id + # "debuerreotype-fixup" will make this an empty file for reproducibility, but for our Docker images it seems more appropriate for it to not exist (since they've never actually been "booted" so having the "first boot" logic trigger if someone were to run systemd in them conceptually makes sense) + ) + + case "$suite" in + sarge) # 3.1 + # for some reason, sarge creates "/var/cache/man/index.db" with some obvious embedded unix timestamps (but if we exclude it, "man" still works properly, so *shrug*) + tarArgs+=( --exclude ./var/cache/man/index.db ) + ;; + + woody) # 3.0 + # woody not only contains "exim", but launches it during our build process and tries to email "root@debuerreotype" (which fails and creates non-reproducibility) + tarArgs+=( --exclude ./var/spool/exim --exclude ./var/log/exim ) + ;; + + potato) # 2.2 + tarArgs+=( + # for some reason, pototo leaves a core dump (TODO figure out why??) + --exclude './core' + # also, it leaves some junk in /tmp (/tmp/fdmount.conf.tmp.XXX) + --exclude './tmp/fdmount.conf.tmp.*' + ) + ;; + + slink) # 2.1 + tarArgs+=( + # same as potato :( + --exclude './tmp/fdmount.conf.tmp.*' + ) + ;; + esac + + debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" + du -hsx "$targetBase.tar.xz" + + sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" + touch_epoch "$targetBase.tar.xz.sha256" + + debuerreotype-chroot "$rootfs" bash -c ' + if ! dpkg-query -W 2> /dev/null; then + # --debian-eol woody has no dpkg-query + dpkg -l + fi + ' > "$targetBase.manifest" + echo "$suite" > "$targetBase.apt-dist" + echo "$dpkgArch" > "$targetBase.dpkg-arch" + echo "$epoch" > "$targetBase.debuerreotype-epoch" + echo "$variant" > "$targetBase.debuerreotype-variant" + debuerreotype-version > "$targetBase.debuerreotype-version" + touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} + + for f in debian_version os-release apt/sources.list; do + targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" + if [ -e "$rootfs/etc/$f" ]; then + # /etc/os-release does not exist in --debian-eol squeeze, for example (hence the existence check) + cp "$rootfs/etc/$f" "$targetFile" + touch_epoch "$targetFile" + fi + done +} + +for rootfs in "$rootfsDir"*/; do + rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... + + du -hsx "$rootfs" + + variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... + variant="${variant#rootfs}" # "", "-slim", ... + variant="${variant#-}" # "", "slim", ... + + variantDir="$tmpOutputDir/$variant" + mkdir -p "$variantDir" + + targetBase="$variantDir/rootfs" + + create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" +done + +if [ -n "$codenameCopy" ]; then + codenameDir="$archDir/$codename" + mkdir -p "$codenameDir" + tar -cC "$tmpOutputDir" --exclude='**/rootfs.*' . | tar -xC "$codenameDir" + + for rootfs in "$rootfsDir"*/; do + rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... + + variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... + variant="${variant#rootfs}" # "", "-slim", ... + variant="${variant#-}" # "", "slim", ... + + variantDir="$codenameDir/$variant" + targetBase="$variantDir/rootfs" + + # point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite) + debuerreotype-debian-sources-list --snapshot "${sourcesListArgs[@]}" "$rootfs" "$codename" + + create_artifacts "$targetBase" "$rootfs" "$codename" "$variant" + done +fi + +user="$(stat --format '%u' "$outputDir")" +group="$(stat --format '%g' "$outputDir")" +tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" diff -Nru debuerreotype-0.10/examples/oci-image.sh debuerreotype-0.14/examples/oci-image.sh --- debuerreotype-0.10/examples/oci-image.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/examples/oci-image.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# create https://github.com/opencontainers/image-spec/blob/v1.0.1/image-layout.md (+ Docker's "manifest.json") from the output of "debian.sh" +# the resulting file is suitable for "ctr image import" or "docker load" + +# (this can *technically* run via "docker-run.sh", but IMO it's much easier to run unprivileged on the host) +# RUN apt-get update \ +# && apt-get install -y jq pigz \ +# && rm -rf /var/lib/apt/lists/* + +thisDir="$(readlink -vf "$BASH_SOURCE")" +thisDir="$(dirname "$thisDir")" + +if [ -x "$thisDir/../scripts/debuerreotype-init" ]; then + debuerreotypeScriptsDir="$(dirname "$thisDir")/scripts" +else + debuerreotypeScriptsDir="$(which debuerreotype-init)" + debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" + debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" +fi + +source "$debuerreotypeScriptsDir/.constants.sh" \ + --flags 'meta:' \ + -- \ + ' ' \ + 'out/oci-unstable-slim.tar out/20210511/amd64/unstable/slim' + +eval "$dgetopt" +meta= +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --meta) meta="$1"; shift ;; + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +targetFile="${1:-}"; shift || eusage 'missing target-file' # "something.tar" +sourceDir="${1:-}"; shift || eusage 'missing source-directory' # "out/YYYYMMDD/ARCH/SUITE{,/slim}" + +targetFile="$(readlink -vf "$targetFile")" +if [ -n "$meta" ]; then + meta="$(readlink -vf "$meta")" +fi +sourceDir="$(readlink -ve "$sourceDir")" + +tempDir="$(mktemp -d)" +trap "$(printf 'rm -rf %q' "$tempDir")" EXIT + +mkdir -p "$tempDir/oci/blobs/sha256" +jq -ncS '{ imageLayoutVersion: "1.0.0" }' > "$tempDir/oci/oci-layout" + +version="$(< "$sourceDir/rootfs.debuerreotype-version")" +epoch="$(< "$sourceDir/rootfs.debuerreotype-epoch")" +iso8601="$(date --date="@$epoch" '+%Y-%m-%dT%H:%M:%SZ')" +export version epoch iso8601 + +if [ -s "$sourceDir/rootfs.apt-dist" ]; then + suite="$(< "$sourceDir/rootfs.apt-dist")" + # TODO remove this fallback once debuerreotype 0.13 is released and we can safely assume "rootfs.apt-dist" exists +else + suite="$(awk '$1 == "deb" { print $3; exit }' "$sourceDir/rootfs.sources-list")" +fi +export suite + +if [ -s "$sourceDir/rootfs.debuerreotype-variant" ]; then + variant="$(< "$sourceDir/rootfs.debuerreotype-variant")" + # TODO remove this fallback once debuerreotype 0.13 is released and we can safely assume "rootfs.debuerreotype-variant" exists +else + dirBase="$(basename "$sourceDir")" + case "$dirBase" in + slim) variant="$dirBase" ;; + "$suite") variant='' ;; + *) echo >&2 "error: unknown variant: '$variant'"; exit 1 ;; + esac +fi + +if [ -s "$sourceDir/rootfs.dpkg-arch" ]; then + dpkgArch="$(< "$sourceDir/rootfs.dpkg-arch")" + # TODO remove these fallbacks once debuerreotype 0.13 is released and we can safely assume "rootfs.dpkg-arch" exists +elif [ -n "$variant" ]; then # xxx/YYYYMMDD/ARCH/SUITE/slim + dpkgArch="$(cd "$sourceDir/../.." && basename "$PWD")" +else # xxx/YYYYMMDD/ARCH/SUITE + dpkgArch="$(cd "$sourceDir/.." && basename "$PWD")" +fi +unset goArch +goArm= +case "$dpkgArch" in + amd64 | arm64 | s390x | riscv64) goArch="$dpkgArch" ;; + armel | arm) goArch='arm'; goArm='5' ;; + armhf) goArch='arm'; if grep -qi raspbian "$sourceDir/rootfs.os-release"; then goArm='6'; else goArm='7'; fi ;; + i386) goArch='386' ;; + mips64el | ppc64el) goArch="${dpkgArch%el}le" ;; + *) echo >&2 "error: unknown dpkg architecture: '$dpkgArch'"; exit 1 ;; +esac +unset bashbrewArch +case "$goArch" in + 386) bashbrewArch='i386' ;; + amd64 | mips64le | ppc64le | riscv64 | s390x) bashbrewArch="$goArch" ;; + arm) bashbrewArch="${goArch}32v${goArm}" ;; + arm64) bashbrewArch="${goArch}v8" ;; + *) echo >&2 "error: unknown Go architecture: '$goArch'"; exit 1 ;; +esac +export dpkgArch goArch goArm bashbrewArch + +osID="$(id="$(grep -E '^ID=' "$sourceDir/rootfs.os-release")" && eval "$id" && echo "${ID:-}")" || : # "debian", "raspbian", "ubuntu", etc +: "${osID:=debian}" # if for some reason the above fails, fall back to "debian" + +echo >&2 "processing $osID '$suite'${variant:+", variant '$variant'"}, architecture '$dpkgArch' ('$bashbrewArch')" + +_sha256() { + sha256sum "$@" | cut -d' ' -f1 +} + +echo >&2 "decompressing rootfs (xz) ..." + +xz --decompress --threads=0 --stdout "$sourceDir/rootfs.tar.xz" > "$tempDir/rootfs.tar" +diffId="$(_sha256 "$tempDir/rootfs.tar")" +export diffId="sha256:$diffId" + +echo >&2 "recompressing rootfs (gzip) ..." + +pigz --best --no-time "$tempDir/rootfs.tar" +rootfsSize="$(stat --format='%s' "$tempDir/rootfs.tar.gz")" +rootfsSha256="$(_sha256 "$tempDir/rootfs.tar.gz")" +export rootfsSize rootfsSha256 +mv "$tempDir/rootfs.tar.gz" "$tempDir/oci/blobs/sha256/$rootfsSha256" + +script='debian.sh' +if [ -x "$thisDir/$osID.sh" ]; then + script="$osID.sh" +fi +export script + +echo >&2 "generating config ..." + +# https://github.com/opencontainers/image-spec/blob/v1.0.1/config.md +jq -ncS ' + { + config: { + Env: [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ], + Entrypoint: [], + Cmd: [ "bash" ], + }, + created: env.iso8601, + history: [ + { + created: env.iso8601, + created_by: ( + "# " + env.script + + if env.script != "raspbian.sh" then + " --arch " + (env.dpkgArch | @sh) + else "" end + + " out/ " + + (env.suite | @sh) + + if env.script == "debian.sh" then + " " + else + " # " + end + + ("@" + env.epoch | @sh) + ), + comment: ( "debuerreotype " + env.version ), + } + ], + rootfs: { + type: "layers", + diff_ids: [ env.diffId ], + }, + os: "linux", + architecture: env.goArch, + } + + if env.goArch == "arm64" then + { variant: "v8" } + elif env.goArch == "arm" then + { variant: ( "v" + env.goArm ) } + else + {} + end +' > "$tempDir/config.json" +configSize="$(stat --format='%s' "$tempDir/config.json")" +configSha256="$(_sha256 "$tempDir/config.json")" +export configSize configSha256 +mv "$tempDir/config.json" "$tempDir/oci/blobs/sha256/$configSha256" + +# https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md +jq -ncS ' + { + schemaVersion: 2, + mediaType: "application/vnd.docker.distribution.manifest.v2+json", + config: { + mediaType: "application/vnd.docker.container.image.v1+json", + size: (env.configSize | tonumber), + digest: ( "sha256:" + env.configSha256 ), + }, + layers: [ + { + mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + size: (env.rootfsSize | tonumber), + digest: ( "sha256:" + env.rootfsSha256 ), + } + ], + } +' > "$tempDir/manifest.json" +manifestSize="$(stat --format='%s' "$tempDir/manifest.json")" +manifestSha256="$(_sha256 "$tempDir/manifest.json")" +export manifestSize manifestSha256 +mv "$tempDir/manifest.json" "$tempDir/oci/blobs/sha256/$manifestSha256" + +export repo="$bashbrewArch/$osID" # "amd64/debian", "arm32v6/raspbian", etc. +export tag="$suite${variant:+-$variant}" # "buster", "buster-slim", etc. +export image="$repo:$tag" + +# https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md +jq -ncS ' + { + schemaVersion: 2, + manifests: [ + { + mediaType: "application/vnd.docker.distribution.manifest.v2+json", + size: (env.manifestSize | tonumber), + digest: ( "sha256:" + env.manifestSha256 ), + annotations: { + "io.containerd.image.name": env.image, + "org.opencontainers.image.ref.name": env.tag, + }, + } + ], + } +' > "$tempDir/oci/index.json" + +# Docker's "manifest.json" so that we can "docker load" the result of this script too +jq -ncS ' + [ + { + Config: ( "blobs/sha256/" + env.configSha256 ), + Layers: [ "blobs/sha256/" + env.rootfsSha256 ], + RepoTags: [ env.image ], + } + ] +' > "$tempDir/oci/manifest.json" + +echo >&2 "fixing timestamps ..." + +find "$tempDir/oci" \ + -newermt "@$epoch" \ + -exec touch --no-dereference --date="@$epoch" '{}' + + +echo >&2 "generating tarball ($targetFile) ..." + +tar --create \ + --auto-compress \ + --directory "$tempDir/oci" \ + --file "$targetFile" \ + --numeric-owner --owner 1000:1000 \ + --sort name \ + . +touch --no-dereference --date="@$epoch" "$targetFile" + +platform="$(jq -c 'with_entries(select(.key == ([ "os", "architecture", "variant" ][])))' "$tempDir/oci/blobs/sha256/$configSha256")" +jq -n --argjson platform "$platform" ' + { + image: env.image, + repo: env.repo, + tag: env.tag, + id: ( "sha256:" + env.configSha256 ), + digest: ( "sha256:" + env.manifestSha256 ), + platform: $platform, + } +' > "$tempDir/meta.json" +touch --no-dereference --date="@$epoch" "$tempDir/meta.json" +if [ -n "$meta" ]; then + cp -a "$tempDir/meta.json" "$meta" +fi + +jq >&2 . "$tempDir/meta.json" diff -Nru debuerreotype-0.10/examples/raspbian.sh debuerreotype-0.14/examples/raspbian.sh --- debuerreotype-0.10/examples/raspbian.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/examples/raspbian.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# # https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/ +# RUN wget -O raspbian.deb 'https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring-udeb_20120528.2_all.udeb' \ +# && apt-get install -y ./raspbian.deb \ +# && rm raspbian.deb + +debuerreotypeScriptsDir="$(which debuerreotype-init)" +debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" +debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" + +source "$debuerreotypeScriptsDir/.constants.sh" \ + -- \ + ' ' \ + 'output stretch' + +eval "$dgetopt" +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +outputDir="${1:-}"; shift || eusage 'missing output-dir' +suite="${1:-}"; shift || eusage 'missing suite' + +set -x + +outputDir="$(readlink -ve "$outputDir")" + +tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" +trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT + +export TZ='UTC' LC_ALL='C' + +dpkgArch='armhf' + +exportDir="$tmpDir/output" +archDir="$exportDir/raspbian/$dpkgArch" +tmpOutputDir="$archDir/$suite" + +#mirror='http://archive.raspbian.org/raspbian' +mirror='http://mirrordirector.raspbian.org/raspbian' +# (https://www.raspbian.org/RaspbianMirrors#The_mirror_redirection_system) + +initArgs=( + --arch "$dpkgArch" + --non-debian +) + +export GNUPGHOME="$tmpDir/gnupg" +mkdir -p "$GNUPGHOME" +keyring='/usr/share/keyrings/raspbian-archive-keyring.gpg' +if [ ! -s "$keyring" ]; then + # since we're using mirrors, we ought to be more explicit about download verification + keyUrl='https://archive.raspbian.org/raspbian.public.key' + ( + set +x + echo >&2 + echo >&2 "WARNING: missing '$keyring' (from 'raspbian-archive-keyring' package)" + echo >&2 " downloading '$keyUrl' (without verification beyond TLS)!" + echo >&2 + ) + sleep 5 + keyring="$tmpDir/raspbian-archive-keyring.gpg" + wget -O "$keyring.asc" "$keyUrl" + gpg --batch --no-default-keyring --keyring "$keyring" --import "$keyring.asc" + rm -f "$keyring.asc" +fi +initArgs+=( --keyring "$keyring" ) + +mkdir -p "$tmpOutputDir" + +if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then + gpgv \ + --keyring "$keyring" \ + --output "$tmpOutputDir/Release" \ + "$tmpOutputDir/InRelease" + [ -s "$tmpOutputDir/Release" ] +elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then + rm -f "$tmpOutputDir/InRelease" # remove wget leftovers + gpgv \ + --keyring "$keyring" \ + "$tmpOutputDir/Release.gpg" \ + "$tmpOutputDir/Release" + [ -s "$tmpOutputDir/Release" ] +else + echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" + exit 1 +fi + +initArgs+=( + # disable merged-usr (for now?) due to the following compelling arguments: + # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) + # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) + # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 + --no-merged-usr +) + +rootfsDir="$tmpDir/rootfs" +debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror" + +debuerreotype-minimizing-config "$rootfsDir" + +# TODO do we need to update sources.list here? (security?) +debuerreotype-apt-get "$rootfsDir" update -qq + +debuerreotype-recalculate-epoch "$rootfsDir" +epoch="$(< "$rootfsDir/debuerreotype-epoch")" +touch_epoch() { + while [ "$#" -gt 0 ]; do + local f="$1"; shift + touch --no-dereference --date="@$epoch" "$f" + done +} + +aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" +if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then + debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq +else + debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq +fi + +# copy the rootfs to create other variants +mkdir "$rootfsDir"-slim +tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim + +# for historical reasons (related to their usefulness in debugging non-working container networking in container early days before "--network container:xxx"), Debian 10 and older non-slim images included both "ping" and "ip" above "minbase", but in 11+ (Bullseye), that will no longer be the case and we will instead be a faithful minbase again :D +epoch2021="$(date --date '2021-01-01 00:00:00' +%s)" +if [ "$epoch" -lt "$epoch2021" ] || { isDebianBusterOrOlder="$([ -f "$rootfsDir/etc/os-release" ] && source "$rootfsDir/etc/os-release" && [ -n "${VERSION_ID:-}" ] && [ "${VERSION_ID%%.*}" -le 10 ] && echo 1)" && [ -n "$isDebianBusterOrOlder" ]; }; then + # prefer iproute2 if it exists + iproute=iproute2 + if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then + # poor wheezy + iproute=iproute + fi + ping=iputils-ping + noInstallRecommends='--no-install-recommends' + debuerreotype-apt-get "$rootfsDir" install -y $noInstallRecommends $ping $iproute +fi + +debuerreotype-slimify "$rootfsDir"-slim + +create_artifacts() { + local targetBase="$1"; shift + local rootfs="$1"; shift + local suite="$1"; shift + local variant="$1"; shift + + local tarArgs=() + + debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" + du -hsx "$targetBase.tar.xz" + + sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" + touch_epoch "$targetBase.tar.xz.sha256" + + debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" + echo "$suite" > "$targetBase.apt-dist" + echo "$dpkgArch" > "$targetBase.dpkg-arch" + echo "$epoch" > "$targetBase.debuerreotype-epoch" + echo "$variant" > "$targetBase.debuerreotype-variant" + debuerreotype-version > "$targetBase.debuerreotype-version" + touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} + + for f in debian_version os-release apt/sources.list; do + targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" + if [ -e "$rootfs/etc/$f" ]; then + cp "$rootfs/etc/$f" "$targetFile" + touch_epoch "$targetFile" + fi + done +} + +for rootfs in "$rootfsDir"*/; do + rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... + + du -hsx "$rootfs" + + variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... + variant="${variant#rootfs}" # "", "-slim", ... + variant="${variant#-}" # "", "slim", ... + + variantDir="$tmpOutputDir/$variant" + mkdir -p "$variantDir" + + targetBase="$variantDir/rootfs" + + create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" +done + +user="$(stat --format '%u' "$outputDir")" +group="$(stat --format '%g' "$outputDir")" +tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" diff -Nru debuerreotype-0.10/examples/steamos.sh debuerreotype-0.14/examples/steamos.sh --- debuerreotype-0.10/examples/steamos.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/examples/steamos.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# # https://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/?C=M&O=D +# RUN wget -O valve.deb 'https://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/valve-archive-keyring_0.6+bsosc2_all.deb' \ +# && apt-get install -y ./valve.deb \ +# && rm valve.deb + +debuerreotypeScriptsDir="$(which debuerreotype-init)" +debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" +debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" + +source "$debuerreotypeScriptsDir/.constants.sh" \ + --flags 'arch:' \ + -- \ + '[--arch=] ' \ + 'output' + +eval "$dgetopt" +arch= +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +outputDir="${1:-}"; shift || eusage 'missing output-dir' +suite="${1:-brewmaster}" # http://repo.steampowered.com/steamos/dists/ + +set -x + +outputDir="$(readlink -ve "$outputDir")" + +tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" +trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT + +export TZ='UTC' LC_ALL='C' + +dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" + +exportDir="$tmpDir/output" +archDir="$exportDir/steamos/$dpkgArch" +tmpOutputDir="$archDir/$suite" + +mirror='http://repo.steampowered.com/steamos' + +initArgs=( + --arch "$dpkgArch" + --non-debian + + --debootstrap-script jessie + --include valve-archive-keyring + --exclude debian-archive-keyring +) + +keyring='/usr/share/keyrings/valve-archive-keyring.gpg' +if [ -f "$keyring" ]; then + initArgs+=( --keyring "$keyring" ) +else + initArgs+=( --no-check-gpg ) +fi + +mkdir -p "$tmpOutputDir" + +if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then + gpgv \ + --keyring "$keyring" \ + --output "$tmpOutputDir/Release" \ + "$tmpOutputDir/InRelease" + [ -s "$tmpOutputDir/Release" ] +elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then + rm -f "$tmpOutputDir/InRelease" # remove wget leftovers + gpgv \ + --keyring "$keyring" \ + "$tmpOutputDir/Release.gpg" \ + "$tmpOutputDir/Release" + [ -s "$tmpOutputDir/Release" ] +else + rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers + echo >&2 "warning: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" +fi + +initArgs+=( + # disable merged-usr (for now?) due to the following compelling arguments: + # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) + # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) + # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 + --no-merged-usr +) + +rootfsDir="$tmpDir/rootfs" +debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror" + +debuerreotype-minimizing-config "$rootfsDir" + +echo "deb $mirror $suite main contrib non-free" | tee "$rootfsDir/etc/apt/sources.list" +debuerreotype-apt-get "$rootfsDir" update -qq + +debuerreotype-recalculate-epoch "$rootfsDir" +epoch="$(< "$rootfsDir/debuerreotype-epoch")" +touch_epoch() { + while [ "$#" -gt 0 ]; do + local f="$1"; shift + touch --no-dereference --date="@$epoch" "$f" + done +} +touch_epoch "$rootfsDir/etc/apt/sources.list" + +aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" +if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then + debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq +else + debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq +fi + +# copy the rootfs to create other variants +mkdir "$rootfsDir"-slim +tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim + +# prefer iproute2 if it exists +iproute=iproute2 +if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then + # poor wheezy + iproute=iproute +fi +debuerreotype-apt-get "$rootfsDir" install -y --no-install-recommends iputils-ping $iproute + +debuerreotype-slimify "$rootfsDir"-slim + +create_artifacts() { + local targetBase="$1"; shift + local rootfs="$1"; shift + local suite="$1"; shift + local variant="$1"; shift + + local tarArgs=() + + debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" + du -hsx "$targetBase.tar.xz" + + sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" + touch_epoch "$targetBase.tar.xz.sha256" + + debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" + echo "$suite" > "$targetBase.apt-dist" + echo "$dpkgArch" > "$targetBase.dpkg-arch" + echo "$epoch" > "$targetBase.debuerreotype-epoch" + echo "$variant" > "$targetBase.debuerreotype-variant" + debuerreotype-version > "$targetBase.debuerreotype-version" + touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} + + for f in debian_version os-release apt/sources.list; do + targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" + if [ -e "$rootfs/etc/$f" ]; then + cp "$rootfs/etc/$f" "$targetFile" + touch_epoch "$targetFile" + fi + done +} + +for rootfs in "$rootfsDir"*/; do + rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... + + du -hsx "$rootfs" + + variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... + variant="${variant#rootfs}" # "", "-slim", ... + variant="${variant#-}" # "", "slim", ... + + variantDir="$tmpOutputDir/$variant" + mkdir -p "$variantDir" + + targetBase="$variantDir/rootfs" + + create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" +done + +user="$(stat --format '%u' "$outputDir")" +group="$(stat --format '%g' "$outputDir")" +tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" diff -Nru debuerreotype-0.10/examples/ubuntu.sh debuerreotype-0.14/examples/ubuntu.sh --- debuerreotype-0.10/examples/ubuntu.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/examples/ubuntu.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# RUN apt-get update \ +# && apt-get install -y ubuntu-keyring \ +# && rm -rf /var/lib/apt/lists/* + +debuerreotypeScriptsDir="$(which debuerreotype-init)" +debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")" +debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")" + +source "$debuerreotypeScriptsDir/.constants.sh" \ + --flags 'arch:' \ + -- \ + '[--arch=] ' \ + 'output xenial +--arch arm64 output bionic' + +eval "$dgetopt" +arch= +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +outputDir="${1:-}"; shift || eusage 'missing output-dir' +suite="${1:-}"; shift || eusage 'missing suite' + +set -x + +outputDir="$(readlink -ve "$outputDir")" + +tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")" +trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT + +export TZ='UTC' LC_ALL='C' + +dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" + +exportDir="$tmpDir/output" +archDir="$exportDir/ubuntu/$dpkgArch" +tmpOutputDir="$archDir/$suite" + +case "$dpkgArch" in + amd64 | i386) + mirror='http://archive.ubuntu.com/ubuntu' + secmirror='http://security.ubuntu.com/ubuntu' + ;; + + *) + mirror='http://ports.ubuntu.com/ubuntu-ports' + secmirror="$mirror" # no separate security mirror for ports + ;; +esac + +initArgs=( + --arch "$dpkgArch" + --non-debian +) + +keyring='/usr/share/keyrings/ubuntu-archive-keyring.gpg' +initArgs+=( --keyring "$keyring" ) + +mkdir -p "$tmpOutputDir" + +if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then + gpgv \ + --keyring "$keyring" \ + --output "$tmpOutputDir/Release" \ + "$tmpOutputDir/InRelease" + [ -s "$tmpOutputDir/Release" ] +elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then + rm -f "$tmpOutputDir/InRelease" # remove wget leftovers + gpgv \ + --keyring "$keyring" \ + "$tmpOutputDir/Release.gpg" \ + "$tmpOutputDir/Release" + [ -s "$tmpOutputDir/Release" ] +else + rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers + echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')" + exit 1 +fi + +initArgs+=( + # disable merged-usr (for now?) due to the following compelling arguments: + # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc) + # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still) + # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406 + --no-merged-usr +) + +rootfsDir="$tmpDir/rootfs" +debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror" + +debuerreotype-minimizing-config "$rootfsDir" + +# setup "proper" sources.list +tee "$rootfsDir/etc/apt/sources.list" <<-EOS + deb $mirror $suite main restricted universe multiverse + deb $mirror $suite-updates main restricted universe multiverse + deb $mirror $suite-backports main restricted universe multiverse + deb $secmirror $suite-security main restricted universe multiverse +EOS +# TODO make components list a script flag? backports? +debuerreotype-apt-get "$rootfsDir" update -qq + +debuerreotype-recalculate-epoch "$rootfsDir" +epoch="$(< "$rootfsDir/debuerreotype-epoch")" +touch_epoch() { + while [ "$#" -gt 0 ]; do + local f="$1"; shift + touch --no-dereference --date="@$epoch" "$f" + done +} +touch_epoch "$rootfsDir/etc/apt/sources.list" + +aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")" +if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then + debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq +else + debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq +fi + +# copy the rootfs to create other variants +mkdir "$rootfsDir"-slim +tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim + +# prefer iproute2 if it exists +iproute=iproute2 +if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then + # poor wheezy + iproute=iproute +fi +debuerreotype-apt-get "$rootfsDir" install -y --no-install-recommends iputils-ping $iproute + +debuerreotype-slimify "$rootfsDir"-slim + +create_artifacts() { + local targetBase="$1"; shift + local rootfs="$1"; shift + local suite="$1"; shift + local variant="$1"; shift + + local tarArgs=() + + debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz" + du -hsx "$targetBase.tar.xz" + + sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256" + touch_epoch "$targetBase.tar.xz.sha256" + + debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" + echo "$suite" > "$targetBase.apt-dist" + echo "$dpkgArch" > "$targetBase.dpkg-arch" + echo "$epoch" > "$targetBase.debuerreotype-epoch" + echo "$variant" > "$targetBase.debuerreotype-variant" + debuerreotype-version > "$targetBase.debuerreotype-version" + touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*} + + for f in debian_version os-release apt/sources.list; do + targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" + if [ -e "$rootfs/etc/$f" ]; then + cp "$rootfs/etc/$f" "$targetFile" + touch_epoch "$targetFile" + fi + done +} + +for rootfs in "$rootfsDir"*/; do + rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ... + + du -hsx "$rootfs" + + variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ... + variant="${variant#rootfs}" # "", "-slim", ... + variant="${variant#-}" # "", "slim", ... + + variantDir="$tmpOutputDir/$variant" + mkdir -p "$variantDir" + + targetBase="$variantDir/rootfs" + + create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" +done + +user="$(stat --format '%u' "$outputDir")" +group="$(stat --format '%g' "$outputDir")" +tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir" diff -Nru debuerreotype-0.10/.github/workflows/ci.yml debuerreotype-0.14/.github/workflows/ci.yml --- debuerreotype-0.10/.github/workflows/ci.yml 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/.github/workflows/ci.yml 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,72 @@ +name: GitHub CI + +on: + pull_request: + push: + schedule: + - cron: 0 0 * * 0 + +defaults: + run: + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + + https: + name: Ensure no-TLS snapshot usage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Ensure http://snapshot.debian.org (https://github.com/debuerreotype/debuerreotype/pull/119#issuecomment-901457009) + run: | + rm .github/workflows/ci.yml # this file itself will always be a match, but it's the only valid one 👀 + ! grep -rn 'https://snapshot.debian.org' + + test: + strategy: + matrix: + include: + - { SUITE: stable, CODENAME: jessie, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: a712fb9b228f107a0d65e836127a377857ab70388f2e23d3b583ac56f8f5a75b } + - { SUITE: jessie, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: a712fb9b228f107a0d65e836127a377857ab70388f2e23d3b583ac56f8f5a75b } + - { SUITE: testing, CODENAME: stretch, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 56adf91c1aca0b52ba7c818f4cdc93e47a6c2c93266296d7e0e108ddead825b4 } + - { SUITE: stretch, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 56adf91c1aca0b52ba7c818f4cdc93e47a6c2c93266296d7e0e108ddead825b4 } + - { SUITE: unstable, CODENAME: sid, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 87f46eeb98d44ff5742d87112d9cc45e51dbb1204d60cb4136b51f0edfce061f } + - { SUITE: sid, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 87f46eeb98d44ff5742d87112d9cc45e51dbb1204d60cb4136b51f0edfce061f } + - { SUITE: oldstable, CODENAME: wheezy, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 52f942d1d0f5ed2d41cb81b63091dc5da0872da94b80a684937cdab7fc57ac06 } + - { SUITE: wheezy, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 52f942d1d0f5ed2d41cb81b63091dc5da0872da94b80a684937cdab7fc57ac06 } + + # EOL suites testing + - { SUITE: eol, CODENAME: etch, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 893d436a060f2536f70efbdfd2e2952cf311eada558f858e6190c80b323b783e } + - { SUITE: eol, CODENAME: lenny, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: c263084bc482b1538512eb091095dd30cf55a6873b989aeee9d4e148f2f3fafa } + - { SUITE: eol, CODENAME: woody, ARCH: i386, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: f80833896e141fbfebf8c91e79da2ccca1bdeb8f8ecc4e05dd33531c32857e0f } + - { SUITE: eol, CODENAME: jessie, TIMESTAMP: "2021-03-01T00:00:00Z", SHA256: a151dd2b4209e152c9323778e3d1016b26e89881df7dd3d81a39282cbd37bdae } + + # qemu-debootstrap testing + - { ARCH: arm64, SUITE: jessie, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: e0bbe759a5df8a3201c3bfc984a91d7ee46bb7cb939d7c06275ff374043ed731 } + - { ARCH: sh4, SUITE: unstable, CODENAME: "", TIMESTAMP: "2022-02-01T00:00:00Z", SHA256: 82949d1099c18dd13bd2d732238d94ae70d03abc50afc6950baee257cce37fd1 } + - { ARCH: riscv64, SUITE: unstable, CODENAME: "", TIMESTAMP: "2022-02-01T00:00:00Z", SHA256: 2a4accea7aef15d4979435ec16f898dcb784330b7b4991fc9ca38b11f87035f5 } + + # a few entries for "today" to try and catch issues like https://github.com/debuerreotype/debuerreotype/issues/41 sooner + - { SUITE: unstable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" } + - { SUITE: stable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" } + - { SUITE: oldstable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" } + + - { DISTRO: ubuntu, SUITE: bionic } + - { DISTRO: ubuntu, SUITE: focal } + fail-fast: false + name: Test ${{ matrix.DISTRO && format('{0} ', matrix.DISTRO) }}${{ matrix.SUITE }}${{ matrix.CODENAME && format(' ({0})', matrix.CODENAME) }}${{ matrix.ARCH && format(' [{0}]', matrix.ARCH) }}${{ matrix.TIMESTAMP && format(' at {0}', matrix.TIMESTAMP) }} + runs-on: ubuntu-20.04 + env: ${{ matrix }} + steps: + - uses: actions/checkout@v2 + - name: Prepare Environment + run: | + sudo apt-get update -qq + sudo apt-get install -yqq binfmt-support qemu-user-static + docker run -d --name squignix --restart always tianon/squignix + git clone --depth 1 https://github.com/tianon/pgp-happy-eyeballs.git ~/phe + ~/phe/hack-my-builds.sh + rm -rf ~/phe + - name: Build + run: | + "./.validate-${DISTRO:-debian}.sh" diff -Nru debuerreotype-0.10/raspbian.sh debuerreotype-0.14/raspbian.sh --- debuerreotype-0.10/raspbian.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/raspbian.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,177 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" -source "$thisDir/scripts/.constants.sh" \ - --flags 'no-build' \ - -- \ - '[--no-build] ' \ - 'output stretch' - -eval "$dgetopt" -build=1 -while true; do - flag="$1"; shift - dgetopt-case "$flag" - case "$flag" in - --no-build) build= ;; # for skipping "docker build" - --) break ;; - *) eusage "unknown flag '$flag'" ;; - esac -done - -outputDir="${1:-}"; shift || eusage 'missing output-dir' -suite="${1:-}"; shift || eusage 'missing suite' - -mkdir -p "$outputDir" -outputDir="$(readlink -f "$outputDir")" - -securityArgs=( - --cap-add SYS_ADMIN - --cap-drop SETFCAP -) -if docker info | grep -q apparmor; then - # AppArmor blocks mount :) - securityArgs+=( - --security-opt apparmor=unconfined - ) -fi - -ver="$("$thisDir/scripts/debuerreotype-version")" -ver="${ver%% *}" -dockerImage="debuerreotype/debuerreotype:$ver" -[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" - -raspbianDockerImage="$dockerImage-raspbian" -[ -z "$build" ] || docker build -t "$raspbianDockerImage" - <<-EODF - FROM $dockerImage - RUN wget -O raspbian.deb 'https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring-udeb_20120528.2_all.udeb' \\ - && apt install -y ./raspbian.deb \\ - && rm raspbian.deb -EODF - -docker run \ - --rm \ - "${securityArgs[@]}" \ - -v /tmp \ - -w /tmp \ - -e suite="$suite" \ - -e TZ='UTC' -e LC_ALL='C' \ - "$raspbianDockerImage" \ - bash -Eeuo pipefail -c ' - set -x - - mirror="http://archive.raspbian.org/raspbian" - - dpkgArch="armhf" - - exportDir="output" - outputDir="$exportDir/raspbian/$dpkgArch/$suite" - - mkdir -p "$outputDir" - wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" - wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" - gpgv \ - --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \ - "$outputDir/Release.gpg" \ - "$outputDir/Release" - - { - debuerreotype-init --non-debian \ - --arch "$dpkgArch" \ - --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \ - --no-merged-usr \ - rootfs "$suite" "$mirror" - - epoch="$(< rootfs/debuerreotype-epoch)" - touch_epoch() { - while [ "$#" -gt 0 ]; do - local f="$1"; shift - touch --no-dereference --date="@$epoch" "$f" - done - } - - debuerreotype-minimizing-config rootfs - debuerreotype-apt-get rootfs update -qq - debuerreotype-apt-get rootfs dist-upgrade -yqq - - # make a couple copies of rootfs so we can create other variants - for variant in slim sbuild; do - mkdir "rootfs-$variant" - tar -cC rootfs . | tar -xC "rootfs-$variant" - done - - # prefer iproute2 if it exists - iproute=iproute2 - if ! debuerreotype-chroot rootfs apt-get install -qq -s iproute2 &> /dev/null; then - # poor wheezy - iproute=iproute - fi - debuerreotype-apt-get rootfs install -y --no-install-recommends iputils-ping $iproute - - debuerreotype-slimify rootfs-slim - - # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild - # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 - # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 - debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot - - create_artifacts() { - local targetBase="$1"; shift - local rootfs="$1"; shift - local suite="$1"; shift - local variant="$1"; shift - - if [ "$variant" != "sbuild" ]; then - debuerreotype-tar "$rootfs" "$targetBase.tar.xz" - else - # sbuild needs "deb-src" entries - debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list - - # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes - # (which are used in sbuild for "--extra-package") - # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - # ... - # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" - # TODO figure out the bug and fix it in APT instead /o\ - - # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") - # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 - debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" - fi - du -hsx "$targetBase.tar.xz" - - sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" - touch_epoch "$targetBase.tar.xz.sha256" - - debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" - echo "$epoch" > "$targetBase.debuerreotype-epoch" - touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" - - for f in debian_version os-release apt/sources.list; do - targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" - cp "$rootfs/etc/$f" "$targetFile" - touch_epoch "$targetFile" - done - } - - for rootfs in rootfs*/; do - rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... - - du -hsx "$rootfs" - - variant="${rootfs#rootfs}" # "", "-slim", ... - variant="${variant#-}" # "", "slim", ... - - variantDir="$outputDir/$variant" - mkdir -p "$variantDir" - - targetBase="$variantDir/rootfs" - - create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" - done - } >&2 - - tar -cC "$exportDir" . - ' | tar -xvC "$outputDir" diff -Nru debuerreotype-0.10/README.md debuerreotype-0.14/README.md --- debuerreotype-0.10/README.md 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/README.md 2022-03-18 19:37:10.000000000 +0000 @@ -1,6 +1,6 @@ # Debuerreotype -[![Build Status](https://travis-ci.org/debuerreotype/debuerreotype.svg?branch=master)](https://travis-ci.org/debuerreotype/debuerreotype/branches) +[![GitHub CI](https://github.com/debuerreotype/debuerreotype/workflows/GitHub%20CI/badge.svg?branch=master&event=push)](https://github.com/debuerreotype/debuerreotype/actions?query=workflow%3A%22GitHub+CI%22+branch%3Amaster) Reproducible, [snapshot](http://snapshot.debian.org)-based Debian rootfs builds (especially for Docker). @@ -33,6 +33,7 @@ | `debuerreotype-minimizing-config` | apply configuration tweaks to make the rootfs minimal and keep it minimal (especially targeted at Docker images, with comments explicitly describing Docker use cases) | | `debuerreotype-slimify` | remove files such as documentation to create an even smaller rootfs (used for creating `slim` variants of the Docker images, for example) | | `debuerreotype-debian-sources-list` | generate an appropriate Debian `sources.list` in the rootfs given a suite (especially for updating `sources.list` to point at deb.debian.org before generating outputs) | +| `debuerreotype-recalculate-epoch` | (esp. for non-Debian) recalculate `debuerreotype-epoch` from `/var/lib/apt/lists/*_{In,}Release` files' `Date:` fields (after updating `sources.list` / `apt-get update`) | | `debuerreotype-fixup` | invoked by `debuerreotype-tar` to fixup timestamps and remove known-bad log files for determinism | | `debuerreotype-tar` | deterministically create a tar file of the rootfs | | `debuerreotype-version` | print out the version of the current `debuerreotype` installation | @@ -41,10 +42,10 @@ The provided `Dockerfile` also includes comments with hints for bootstrapping the environment on a new architecture (which then presumably doesn't have a `debian` Docker base image yet). -Full example: (see [`build.sh`](build.sh) for this in practice) +Full example: (see [`examples/debian.sh`](examples/debian.sh) for this in practice) ```console -$ debuerreotype-init rootfs stretch 2017-01-01T00:00:00Z +$ debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg rootfs stretch 2017-01-01T00:00:00Z I: Retrieving InRelease I: Checking Release signature I: Valid Release signature (key id 126C0D24BD8A2942CC7DF8AC7638D0442B90D010) @@ -89,15 +90,17 @@ $ debuerreotype-debian-sources-list rootfs stretch $ debuerreotype-tar rootfs - | sha256sum -a076d4cd04f68ee117e598a40cc947ad051fc8b063340da015fdceddeb1b0e75 - +e6f10da22f7ab5996f855c85ad5ae38cd786029c57893436c3bb2320f30bc188 - $ # try it! you should get that same sha256sum value! ``` +(As a one-liner via [`docker-run.sh`](docker-run.sh): `./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs stretch 2017-01-01T00:00:00Z; debuerreotype-minimizing-config /tmp/rootfs; debuerreotype-apt-get /tmp/rootfs update -qq; debuerreotype-apt-get /tmp/rootfs dist-upgrade -yqq; debuerreotype-apt-get /tmp/rootfs install -yqq --no-install-recommends inetutils-ping iproute2; debuerreotype-debian-sources-list /tmp/rootfs stretch; debuerreotype-tar /tmp/rootfs - | sha256sum'`) + ## How much have you verified this? Well, I ran the scripts across seven explicit architectures (`amd64`, `arm64`, `armel`, `armhf`, `i386`, `ppc64el`, `s390x`) and eight explicit suites (`oldstable`, `stable`, `testing`, `unstable`, `wheezy`, `jessie`, `stretch`, `sid`) for a timestamp of `2017-05-16T00:00:00Z` (where supported, since `wheezy`/`oldstable` didn't or no longer currently supports some of those architectures), and there were no modifications to any of the tarballs after several runs across several days. -Additionally, Travis runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected. +Additionally, GitHub Actions runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected. From time to time, comments in the files generated by `debuerreotype-minimizing-config` might change (for example), which would obviously result in a different checksum, but a simple [`diffoscope`](https://diffoscope.org/) should be sufficient to verify that the change is benign. diff -Nru debuerreotype-0.10/scripts/.apt-version.sh debuerreotype-0.14/scripts/.apt-version.sh --- debuerreotype-0.10/scripts/.apt-version.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.apt-version.sh 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' @@ -19,12 +19,26 @@ targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] -# scrape our APT version so we can do some basic feature detection (especially to remove unsupported settings on --debian-eol) +package="${1:-apt}" + +# if dpkg-query does not exist, we must be on woody or older, so just assume something ancient (suggested version is the one in woody, since it should be old enough for any "fancy" features we're using this to exclude) +fallback= +case "$package" in + apt) fallback='0.5.4' ;; # woody + dpkg) fallback='1.9.21' ;; # woody +esac + +# scrape package versions so we can do some basic feature detection (especially to remove unsupported settings on --debian-eol) "$thisDir/debuerreotype-chroot" "$targetDir" bash -c ' + package="$1"; shift + fallback="$1"; shift if command -v dpkg-query &> /dev/null; then - dpkg-query --show --showformat "\${Version}\n" apt + dpkg-query --show --showformat "\${Version}\n" "$package" + elif [ -n "$fallback" ]; then + # if dpkg-query does not exist, we must be on woody or older + echo "$fallback" else - # if dpkg-query does not exist, we must be on woody or potato, so just assume something ancient like 0.5.4 (since that is what woody includes and is old enough to cover all our features being excluded) - echo 0.5.4 + echo >&2 "error: missing dpkg-query and no fallback defined in debuerreotype for $package" + exit 1 fi -' +' -- "$package" "$fallback" diff -Nru debuerreotype-0.10/scripts/.constants.sh debuerreotype-0.14/scripts/.constants.sh --- debuerreotype-0.10/scripts/.constants.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.constants.sh 2022-03-18 19:37:10.000000000 +0000 @@ -3,7 +3,7 @@ # constants of the universe export TZ='UTC' LC_ALL='C' umask 0002 -scriptsDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +scriptsDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" self="$(basename "$0")" options="$(getopt -n "$BASH_SOURCE" -o '+' --long 'flags:,flags-short:' -- "$@")" diff -Nru debuerreotype-0.10/scripts/.debian-mirror.sh debuerreotype-0.14/scripts/.debian-mirror.sh --- debuerreotype-0.10/scripts/.debian-mirror.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/scripts/.debian-mirror.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" +source "$thisDir/.constants.sh" \ + --flags 'eol,ports' \ + -- \ + '[--eol] [--ports] ' \ + '--eol 2021-03-01T00:00:00Z jessie amd64 main +2021-03-01T00:00:00Z buster-security arm64 main +--ports 2021-03-01T00:00:00Z sid riscv64 main' + +eval "$dgetopt" +eol= +ports= +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --eol) eol=1 ;; + --ports) ports=1 ;; + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +timestamp="${1:-}"; shift || eusage 'missing timestamp' +suite="${1:-}"; shift || eusage 'missing suite' +arch="${1:-}"; shift || eusage 'missing arch' +component="${1:-}"; shift || eusage 'missing component' + +if [[ "$suite" == *-security ]]; then + target='security' +else + target='standard' +fi + +epoch="$(date --date "$timestamp" '+%s')" + +if [ -z "$ports" ]; then + standardMirrors=( 'http://deb.debian.org/debian' ) + snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" ) +else + standardMirrors=( 'http://deb.debian.org/debian-ports' ) + snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-ports')" ) +fi + +securityMirrors=( 'http://security.debian.org/debian-security' ) +snapshotSecurityMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')" ) + +if [ -n "$eol" ]; then + archiveSnapshotMirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')" + + standardMirrors=( 'http://archive.debian.org/debian' "${standardMirrors[@]}" ) + snapshotStandardMirrors=( "$archiveSnapshotMirror/debian" "${snapshotStandardMirrors[@]}" ) + + securityMirrors=( 'http://archive.debian.org/debian-security' "${securityMirrors[@]}" ) + snapshotSecurityMirrors=( "$archiveSnapshotMirror/debian-security" "${snapshotSecurityMirrors[@]}" ) +fi + +case "$target" in + standard) + nonSnapshotMirrors=( "${standardMirrors[@]}" ) + snapshotMirrors=( "${snapshotStandardMirrors[@]}" ) + ;; + security) + nonSnapshotMirrors=( "${securityMirrors[@]}" ) + snapshotMirrors=( "${snapshotSecurityMirrors[@]}" ) + ;; + *) echo >&2 "error: unknown target: '$target'"; exit 1 ;; +esac + +_find() { + local findSuite="${1:-$suite}" + local i + for i in "${!snapshotMirrors[@]}"; do + local mirror="${nonSnapshotMirrors[$i]}" snapshotMirror="${snapshotMirrors[$i]}" + # http://snapshot.debian.org/archive/debian-archive/20160314T000000Z/debian/dists/squeeze-updates/main/binary-amd64/Packages.gz + if \ + wget --quiet --spider -O /dev/null -o /dev/null "$snapshotMirror/dists/$findSuite/$component/binary-$arch/Packages.xz" \ + || wget --quiet --spider -O /dev/null -o /dev/null "$snapshotMirror/dists/$findSuite/$component/binary-$arch/Packages.gz" \ + ; then + declare -g mirror="$mirror" snapshotMirror="$snapshotMirror" suite="$findSuite" + return 0 + fi + done + if [ "$target" = 'security' ] && [[ "$findSuite" == *-security ]]; then + if _find "${suite%-security}/updates"; then + return 0 + fi + fi + return 1 +} +if ! _find; then + echo >&2 "warning: no apparent '$suite/$component' for '$arch' on any of the following" + for mirror in "${snapshotMirrors[@]}"; do echo >&2 " - $mirror"; done + exit 1 +fi +printf 'mirror=%q\nsnapshotMirror=%q\nfoundSuite=%q\n' "$mirror" "$snapshotMirror" "$suite" diff -Nru debuerreotype-0.10/scripts/.debootstrap-scripts/potato debuerreotype-0.14/scripts/.debootstrap-scripts/potato --- debuerreotype-0.10/scripts/.debootstrap-scripts/potato 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/scripts/.debootstrap-scripts/potato 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,124 @@ +# +# This script was taken from debootstrap version 1.0.125 and then hacked to avoid invoking "init u" in postinst scripts (which tries to send SIGHUP to PID 1). +# + +mirror_style release +download_style apt var-state +default_mirror http://archive.debian.org/debian +force_md5 + +LIBC=libc6 +if [ "$ARCH" = alpha ]; then + LIBC="libc6.1" +fi + +work_out_debs () { + required="base-files base-passwd bash bsdutils debconf-tiny debianutils diff dpkg e2fsprogs fileutils findutils grep gzip hostname ldso libc6 libdb2 libgdbmg1 libncurses5 libnewt0 libpam-modules libpam-runtime libpam0g libpopt0 libreadline4 libstdc++2.10 login makedev mawk modutils mount ncurses-base ncurses-bin passwd perl-5.005-base perl-base procps sed shellutils slang1 sysklogd sysvinit tar textutils update util-linux whiptail" + + base="adduser ae apt base-config elvis-tiny fbset fdutils gettext-base console-data console-tools console-tools-libs libdb2 libwrap0 locales modconf netbase ftp ppp pppconfig pump tasksel tcpd textutils telnet xviddetect" + + without_package () { + echo "$2" | tr ' ' '\n' | grep -v "^$1$" | tr '\n' ' ' + } + + case $ARCH in + "alpha") + required="$(without_package "libc6" "$required") libc6.1" + ;; + "i386") + base="$base fdflush isapnptools lilo mbr pciutils pcmcia-cs psmisc setserial syslinux" + ;; + *) + # other arches may have special needs not yet represented here + # oh well, Potato is old + esac +} + +first_stage_install () { + extract $required + + :> "$TARGET/var/lib/dpkg/status" + echo > "$TARGET/var/lib/dpkg/available" + + setup_etc + echo '# UNCONFIGURED FSTAB FOR BASE SYSTEM' > "$TARGET/etc/fstab" + chown 0:0 "$TARGET/etc/fstab"; chmod 644 "$TARGET/etc/fstab" + + x_feign_install () { + local pkg=$1 + local deb="$(debfor $pkg)" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" + + mkdir -p "$TARGET/var/lib/dpkg/info" + +echo \ +"Package: $pkg +Version: $ver +Status: install ok installed" >> "$TARGET/var/lib/dpkg/status" + + touch "$TARGET/var/lib/dpkg/info/${pkg}.list" + } + + setup_devices + + x_feign_install dpkg + + if [ -e "$TARGET/usr/bin/perl-5.005.dist" ]; then + mv "$TARGET/usr/bin/perl-5.005.dist" "$TARGET/usr/bin/perl-5.005" + fi + if [ ! -e "$TARGET/usr/bin/perl" ]; then + ln -sf perl-5.005 "$TARGET/usr/bin/perl" + fi +} + +second_stage_install () { + x_core_install () { + in_target dpkg --force-depends --install $(debfor "$@") + } + + export DEBIAN_FRONTEND=Noninteractive + + setup_proc + ln "$TARGET/sbin/ldconfig.new" "$TARGET/sbin/ldconfig" + in_target /sbin/ldconfig + + x_core_install base-files base-passwd ldso + x_core_install dpkg + + ln -sf /usr/share/zoneinfo/UTC "$TARGET/etc/localtime" + + # the libc6 and sysvinit postinst scripts want to run "init u" to re-exec sysvinit, but that sends SIGHUP to PID 1, which is undesirable (for hopefully obvious reasons), so we have to get a little bit clever while installing libc6 and later before configuring the rest of the system + x_hack_avoid_postinst_init() { + local postinst + for postinst in "$TARGET"/var/lib/dpkg/info/*.postinst; do + if grep -qE '^[[:space:]]*init[[:space:]]' "$postinst" 2>/dev/null; then + sed -ri -e 's/^([[:space:]]*)(init[[:space:]])/\1: # \2/' "$postinst" + fi + done + } + in_target dpkg --force-depends --unpack $(debfor $LIBC) + x_hack_avoid_postinst_init + in_target dpkg --force-depends --configure $LIBC + + smallyes '' | x_core_install perl-5.005-base + x_core_install mawk + x_core_install debconf-tiny + + in_target dpkg-preconfigure $(debfor $required $base) + + repeatn 5 in_target dpkg --force-depends --unpack $(debfor $required) + + # see above + x_hack_avoid_postinst_init + + mv "$TARGET/sbin/start-stop-daemon" "$TARGET/sbin/start-stop-daemon.REAL" + cp "$TARGET/bin/true" "$TARGET/sbin/start-stop-daemon" + + setup_dselect_method apt + + in_target dpkg --configure --pending --force-configure-any --force-depends + + smallyes '' | repeatn 5 in_target dpkg --force-auto-select --force-overwrite --skip-same-version --install $(debfor $base) + + mv "$TARGET/sbin/start-stop-daemon.REAL" "$TARGET/sbin/start-stop-daemon" +} diff -Nru debuerreotype-0.10/scripts/.debootstrap-scripts/slink debuerreotype-0.14/scripts/.debootstrap-scripts/slink --- debuerreotype-0.10/scripts/.debootstrap-scripts/slink 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/scripts/.debootstrap-scripts/slink 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,101 @@ +# +# This script was taken from debootstrap version 0.2.45-0.2, adapted with several changes to match the corresponding changes to "potato" since it was removed, and then hacked to avoid invoking "init u" in postinst scripts (which tries to send SIGHUP to PID 1). +# + +mirror_style main +download_style apt var-state +default_mirror http://archive.debian.org/debian +force_md5 + +work_out_debs () { + required="base-files base-passwd bash bsdutils debianutils diff dpkg e2fsprogs fileutils findutils grep gzip hostname ldso libdb2 libgdbmg1 libncurses4 ncurses3.4 libpam0g libpam0g-util libpwdb0g libreadlineg2 libstdc++2.9 login makedev mawk modutils mount ncurses-base ncurses-bin newt0.25 passwd perl-base procps sed shellutils slang1 sysklogd sysvinit tar textutils update util-linux whiptail" + + base="adduser ae apt elvis-tiny fbset fdutils console-tools console-tools-libs libdb2 locales modconf netbase textutils telnet" + + case $ARCH in + "i386") + required="$required libc6" + base="$base fdflush isapnptools lilo mbr pciutils psmisc setserial syslinux" + ;; + esac +} + +first_stage_install () { + extract $required + + :> "$TARGET/var/lib/dpkg/status" + echo > "$TARGET/var/lib/dpkg/available" + + setup_etc + echo '# UNCONFIGURED FSTAB FOR BASE SYSTEM' > "$TARGET/etc/fstab" + chown 0:0 "$TARGET/etc/fstab"; chmod 644 "$TARGET/etc/fstab" + + x_feign_install () { + local pkg=$1 + local deb="$(debfor $pkg)" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" + + mkdir -p "$TARGET/var/lib/dpkg/info" + +echo \ +"Package: $pkg +Version: $ver +Status: install ok installed" >> "$TARGET/var/lib/dpkg/status" + + touch "$TARGET/var/lib/dpkg/info/${pkg}.list" + } + + setup_devices + + x_feign_install dpkg + + mv "$TARGET/usr/bin/perl.dist" "$TARGET/usr/bin/perl" +} + +second_stage_install () { + x_core_install () { + in_target dpkg --force-depends --install $(debfor "$@") + } + + setup_proc + ln "$TARGET/sbin/ldconfig.new" "$TARGET/sbin/ldconfig" + in_target /sbin/ldconfig + + x_core_install base-files base-passwd ldso + x_core_install dpkg + + ln -sf /usr/share/zoneinfo/UTC "$TARGET/etc/localtime" + + # the libc6 and sysvinit postinst scripts want to run "init u" to re-exec sysvinit, but that sends SIGHUP to PID 1, which is undesirable (for hopefully obvious reasons), so we have to get a little bit clever while installing libc6 and later before configuring the rest of the system + x_hack_avoid_postinst_init() { + local postinst + for postinst in "$TARGET"/var/lib/dpkg/info/*.postinst; do + if grep -qE '^[[:space:]]*init[[:space:]]' "$postinst" 2>/dev/null; then + sed -ri -e 's/^([[:space:]]*)(init[[:space:]])/\1: # \2/' "$postinst" + fi + done + } + in_target dpkg --force-depends --unpack $(debfor libc6) + x_hack_avoid_postinst_init + in_target dpkg --force-depends --configure libc6 + + x_core_install perl-base + x_core_install mawk + + repeatn 5 in_target dpkg --force-depends --unpack $(debfor $required) + + # see above + x_hack_avoid_postinst_init + + mv "$TARGET/sbin/start-stop-daemon" "$TARGET/sbin/start-stop-daemon.REAL" + cp "$TARGET/bin/true" "$TARGET/sbin/start-stop-daemon" + + setup_dselect_method apt + + in_target dpkg --configure --pending --force-configure-any --force-depends + + smallyes '' | repeatn 5 in_target dpkg --force-auto-select --force-overwrite \ + --skip-same-version --install $(debfor $base) + + mv "$TARGET/sbin/start-stop-daemon.REAL" "$TARGET/sbin/start-stop-daemon" +} diff -Nru debuerreotype-0.10/scripts/debuerreotype-apt-get debuerreotype-0.14/scripts/debuerreotype-apt-get --- debuerreotype-0.10/scripts/debuerreotype-apt-get 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-apt-get 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' arguments' \ 'rootfs update' diff -Nru debuerreotype-0.10/scripts/debuerreotype-chroot debuerreotype-0.14/scripts/debuerreotype-chroot --- debuerreotype-0.10/scripts/debuerreotype-chroot 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-chroot 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [args...]' \ 'rootfs apt-get update' @@ -26,11 +26,13 @@ unshare --mount bash -Eeuo pipefail -c ' [ -n "$targetDir" ] # just to be safe for dir in dev proc sys; do - if [ -e "$targetDir/$dir" ]; then + if [ -d "$targetDir/$dir" ]; then # --debian-eol woody and below have no /sys mount --rbind "/$dir" "$targetDir/$dir" fi done - mount --rbind --read-only /etc/resolv.conf "$targetDir/etc/resolv.conf" + if [ -f "$targetDir/etc/resolv.conf" ]; then + mount --rbind --read-only /etc/resolv.conf "$targetDir/etc/resolv.conf" + fi exec chroot "$targetDir" /usr/bin/env -i PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" TZ="$TZ" LC_ALL="$LC_ALL" SOURCE_DATE_EPOCH="$epoch" "$@" ' -- "$cmd" "$@" diff -Nru debuerreotype-0.10/scripts/debuerreotype-debian-sources-list debuerreotype-0.14/scripts/debuerreotype-debian-sources-list --- debuerreotype-0.10/scripts/debuerreotype-debian-sources-list 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-debian-sources-list 2022-03-18 19:37:10.000000000 +0000 @@ -1,17 +1,18 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ - --flags 'eol,snapshot' \ + --flags 'eol,ports,snapshot' \ --flags 'deb-src' \ -- \ - '[--deb-src] [--eol] [--no-snapshot] ' \ + '[--deb-src] [--eol] [--ports] [--snapshot] ' \ '--snapshot rootfs stretch --eol rootfs wheezy' eval "$dgetopt" eol= +ports= snapshot= debSrc= while true; do @@ -19,6 +20,7 @@ dgetopt-case "$flag" case "$flag" in --eol) eol=1 ;; + --ports) ports=1 ;; --snapshot) snapshot=1 ;; --deb-src) debSrc=1 ;; --) break ;; @@ -32,19 +34,24 @@ epoch="$(< "$targetDir/debuerreotype-epoch")" -standardMirror='http://deb.debian.org/debian' -snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" ) +if [ -z "$ports" ]; then + standardMirrors=( 'http://deb.debian.org/debian' ) + snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" ) +else + standardMirrors=( 'http://deb.debian.org/debian-ports' ) + snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-ports')" ) +fi -securityMirror='http://security.debian.org/debian-security' +securityMirrors=( 'http://security.debian.org/debian-security' ) snapshotSecurityMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')" ) if [ -n "$eol" ]; then archiveSnapshotMirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')" - standardMirror='http://archive.debian.org/debian' + standardMirrors=( 'http://archive.debian.org/debian' "${standardMirrors[@]}" ) snapshotStandardMirrors=( "$archiveSnapshotMirror/debian" "${snapshotStandardMirrors[@]}" ) - securityMirror='http://archive.debian.org/debian-security' + securityMirrors=( 'http://archive.debian.org/debian-security' "${securityMirrors[@]}" ) snapshotSecurityMirrors=( "$archiveSnapshotMirror/debian-security" "${snapshotSecurityMirrors[@]}" ) fi @@ -54,34 +61,30 @@ deb() { local suite="$1"; shift local comp="$1"; shift - local target="$1"; shift # "standard" or "security" - - local nonSnapshotMirror= snapshotMirrors=() - case "$target" in - standard) nonSnapshotMirror="$standardMirror"; snapshotMirrors=( "${snapshotStandardMirrors[@]}" ) ;; - security) nonSnapshotMirror="$securityMirror"; snapshotMirrors=( "${snapshotSecurityMirrors[@]}" ) ;; - *) echo >&2 "error: unknown 'deb' line target: '$target'"; exit 1 ;; - esac - local found= mirror - for mirror in "${snapshotMirrors[@]}"; do - # http://snapshot.debian.org/archive/debian-archive/20160314T000000Z/debian/dists/squeeze-updates/main/binary-amd64/Packages.gz - if wget --quiet --spider -O /dev/null -o /dev/null "$mirror/dists/$suite/$comp/binary-$arch/Packages.gz"; then - found="$mirror" - break - fi - done - if [ -z "$found" ]; then - echo >&2 "warning: no apparent '$suite/$comp' for '$arch' on any of the following; skipping" - for mirror in "${snapshotMirrors[@]}"; do echo >&2 " - $mirror"; done + local mirrorArgs=() + if [ -n "$ports" ]; then + mirrorArgs+=( --ports ) + fi + if [ -n "$eol" ]; then + mirrorArgs+=( --eol ) + fi + mirrorArgs+=( "@$epoch" "$suite" "$arch" "$comp" ) + local mirrors + if ! mirrors="$("$thisDir/.debian-mirror.sh" "${mirrorArgs[@]}")"; then + echo >&2 "skipping '$suite/$comp' ..." return fi + eval "$mirrors" + [ -n "$mirror" ] + [ -n "$snapshotMirror" ] + [ -n "$foundSuite" ] + suite="$foundSuite" if [ -n "$snapshot" ]; then - mirror="$found" + mirror="$snapshotMirror" else - echo "# deb $found $suite $comp" - mirror="$nonSnapshotMirror" + echo "# deb $snapshotMirror $suite $comp" fi echo "deb $mirror $suite $comp" if [ -n "$debSrc" ]; then @@ -92,15 +95,19 @@ # https://github.com/tianon/go-aptsources/blob/e066ed9cd8cd9eef7198765bd00ec99679e6d0be/target.go#L16-L58 { case "$suite" in - sid|unstable|testing) + sid | unstable | testing) deb "$suite" "$comp" standard + if [ -n "$ports" ]; then + # https://www.ports.debian.org/archive + deb unreleased "$comp" standard + fi ;; *) # https://salsa.debian.org/installer-team/apt-setup/tree/d7a642fb5fc76e4f0b684db53984bdb9123f8360/generators - deb "$suite" "$comp" standard # "50mirror" - deb "$suite/updates" "$comp" security # "91security" - deb "$suite-updates" "$comp" standard # "92updates" + deb "$suite" "$comp" standard # "50mirror" + deb "$suite-security" "$comp" security # "91security" + deb "$suite-updates" "$comp" standard # "92updates" # https://wiki.debian.org/SourcesList#Example_sources.list if [ "$suite" = 'squeeze' ]; then diff -Nru debuerreotype-0.10/scripts/debuerreotype-fixup debuerreotype-0.14/scripts/debuerreotype-fixup --- debuerreotype-0.10/scripts/debuerreotype-fixup 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-fixup 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' @@ -21,6 +21,13 @@ epoch="$(< "$targetDir/debuerreotype-epoch")" [ -n "$epoch" ] +if [ -s "$targetDir/etc/machine-id" ]; then + # https://www.freedesktop.org/software/systemd/man/machine-id.html + # > For operating system images which are created once and used on multiple machines, for example for containers or in the cloud, /etc/machine-id should be either missing or an empty file in the generic file system image ... + echo -n > "$targetDir/etc/machine-id" + chmod 0644 "$targetDir/etc/machine-id" +fi + # https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d#diff-de4eef4ab836e5c6c9c1f820a2f624baR709 rm -f \ "$targetDir/var/log/dpkg.log" \ diff -Nru debuerreotype-0.10/scripts/debuerreotype-gpgv-ignore-expiration-config debuerreotype-0.14/scripts/debuerreotype-gpgv-ignore-expiration-config --- debuerreotype-0.10/scripts/debuerreotype-gpgv-ignore-expiration-config 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-gpgv-ignore-expiration-config 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' diff -Nru debuerreotype-0.10/scripts/debuerreotype-init debuerreotype-0.14/scripts/debuerreotype-init --- debuerreotype-0.10/scripts/debuerreotype-init 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-init 2022-03-18 19:37:10.000000000 +0000 @@ -1,13 +1,14 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ - --flags 'debian,debian-eol,non-debian' \ + --flags 'debian,debian-eol,debian-ports,non-debian' \ --flags 'debootstrap:' \ --flags 'debootstrap-script:' \ --flags 'keyring:,arch:,include:,exclude:' \ --flags 'merged-usr,no-merged-usr' \ + --flags 'check-gpg,no-check-gpg' \ -- \ ' ' \ 'rootfs stretch 2017-05-08T00:00:00Z @@ -19,6 +20,7 @@ eval "$dgetopt" nonDebian= debianEol= +debianPorts= debootstrap= script= keyring= @@ -26,13 +28,15 @@ include= exclude= noMergedUsr= +noCheckGpg= while true; do flag="$1"; shift dgetopt-case "$flag" case "$flag" in - --debian) nonDebian= ; debianEol= ;; - --debian-eol) nonDebian= ; debianEol=1 ;; - --non-debian) nonDebian=1; debianEol= ;; + --debian) nonDebian= ;; + --debian-eol) nonDebian= ; debianEol=1 ;; + --debian-ports) nonDebian= ; debianPorts=1 ;; + --non-debian) nonDebian=1 ;; --debootstrap) debootstrap="$1"; shift ;; --debootstrap-script) script="$1"; shift ;; --keyring) keyring="$1"; shift ;; @@ -41,6 +45,8 @@ --exclude) exclude="${exclude:+$exclude,}$1"; shift ;; --merged-usr) noMergedUsr= ;; --no-merged-usr) noMergedUsr=1 ;; + --check-gpg) noCheckGpg= ;; + --no-check-gpg) noCheckGpg=1 ;; --) break ;; *) eusage "unknown flag '$flag'" ;; esac @@ -62,39 +68,55 @@ else mirror="${1:-}"; shift || eusage 'missing mirror' - timestamp="$(wget -qO- "$mirror/dists/$suite/Release" | awk -F ': ' '$1 == "Date" { print $2 }')" + timestamp="$( + { + wget -qO- "$mirror/dists/$suite/InRelease" 2>/dev/null \ + || wget -qO- "$mirror/dists/$suite/Release" 2>/dev/null + } | awk -F ': ' '$1 == "Date" { print $2; exit }' + )" || : + [ -n "$timestamp" ] + + # see "debuerreotype-recalculate-epoch" for a simple way to update this value appropriately after "sources.list" is updated and "apt-get update" has been run (which will be more accurate) fi epoch="$(date --date "$timestamp" '+%s')" export SOURCE_DATE_EPOCH="$epoch" if [ -z "$nonDebian" ]; then - if [ -z "$debianEol" ]; then - mirror="$("$thisDir/.snapshot-url.sh" "@$epoch")" - else - mirrorbase="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')" - mirror="$mirrorbase/debian" + dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}" + mirrorArgs=() + if [ -n "$debianPorts" ]; then + mirrorArgs+=( --ports ) + fi + if [ -n "$debianEol" ]; then + mirrorArgs+=( --eol ) fi + mirrorArgs+=( "@$epoch" "$suite" "$dpkgArch" main ) + mirrors="$("$thisDir/.debian-mirror.sh" "${mirrorArgs[@]}")" + eval "$mirrors" + [ -n "$snapshotMirror" ] + mirror="$snapshotMirror" fi -debootstrapArgs=( - --force-check-gpg -) +debootstrapArgs=() -minbaseSupported="$( - scriptFile="$( - if [ -n "$script" ]; then - readlink -f "$script" - else - cd /usr/share/debootstrap/scripts - readlink -f "$suite" - fi - )" - if grep -q 'minbase' "$scriptFile"; then - echo 1 +if [ -z "$noCheckGpg" ]; then + debootstrapArgs+=( --force-check-gpg ) +else + debootstrapArgs+=( --no-check-gpg ) +fi + +: "${script:=$suite}" +script="$( + if [ -s "$thisDir/.debootstrap-scripts/$script" ]; then + readlink -vf "$thisDir/.debootstrap-scripts/$script" + elif [ -s "/usr/share/debootstrap/scripts/$script" ]; then + readlink -vf "/usr/share/debootstrap/scripts/$script" + else + readlink -vf "$script" fi )" -if [ -n "$minbaseSupported" ]; then +if grep -q 'minbase' "$script"; then # --debian-eol sarge and older do not support minbase debootstrapArgs+=( --variant=minbase ) fi @@ -106,11 +128,15 @@ [ -z "$exclude" ] || debootstrapArgs+=( --exclude="$exclude" ) debootstrapArgs+=( - "$suite" "$targetDir" "$mirror" + "$suite" "$targetDir" "$mirror" "$script" ) -[ -z "$script" ] || debootstrapArgs+=( "$script" ) -: "${debootstrap:=debootstrap}" +unshare-debootstrap() { + # avoid bugs in packages that might leave things mounted ("/run/shm" is a common one that sometimes ends up dangling) + unshare --mount debootstrap "$@" +} + +: "${debootstrap:=unshare-debootstrap}" if ! "$debootstrap" "${debootstrapArgs[@]}"; then if [ -f "$targetDir/debootstrap/debootstrap.log" ]; then echo >&2 @@ -130,7 +156,10 @@ echo "$epoch" > "$targetDir/debuerreotype-epoch" if [ -z "$nonDebian" ]; then - "$thisDir/debuerreotype-debian-sources-list" --snapshot $([ -z "$debianEol" ] || echo '--eol') "$targetDir" "$suite" + "$thisDir/debuerreotype-debian-sources-list" --snapshot \ + $([ -z "$debianEol" ] || echo '--eol') \ + $([ -z "$debianPorts" ] || echo '--ports') \ + "$targetDir" "$suite" "$thisDir/debuerreotype-apt-get" "$targetDir" update -qq fi @@ -150,10 +179,6 @@ ' -- "$include" echo 'debuerreotype' > "$targetDir/etc/hostname" -echo "$epoch" \ - | md5sum \ - | cut -f1 -d' ' \ - > "$targetDir/etc/machine-id" # TODO should we only do this if "/etc/machine-id" already exists? { echo '# https://1.1.1.1 (privacy-focused, highly-available DNS service)' echo 'nameserver 1.1.1.1' @@ -161,7 +186,6 @@ } > "$targetDir/etc/resolv.conf" chmod 0644 \ "$targetDir/etc/hostname" \ - "$targetDir/etc/machine-id" \ "$targetDir/etc/resolv.conf" # fix ownership/permissions on / (otherwise "debootstrap" leaves them as-is which causes reproducibility issues) diff -Nru debuerreotype-0.10/scripts/debuerreotype-minimizing-config debuerreotype-0.14/scripts/debuerreotype-minimizing-config --- debuerreotype-0.10/scripts/debuerreotype-minimizing-config 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-minimizing-config 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' @@ -20,6 +20,7 @@ [ -n "$targetDir" ] aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")" +dpkgVersion="$("$thisDir/.apt-version.sh" "$targetDir" 'dpkg')" # https://github.com/docker/docker/blob/d6f4fe9e38b60f63e429fff7ffced9c26cbf8236/contrib/mkimage/debootstrap#L63-L177 @@ -43,8 +44,9 @@ fi # force dpkg not to call sync() after package extraction (speeding up installs) -if [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ]; then +if [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ] && dpkg --compare-versions "$dpkgVersion" '>=' '1.15.8.6~'; then # --debian-eol lenny and older do not include /etc/dpkg/dpkg.cfg.d + # force-unsafe-io was added in dpkg 1.15.8.6: https://salsa.debian.org/dpkg-team/dpkg/-/commit/929a9c4808c79781469987585f78f07df7f1d484 cat > "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" <<-'EOF' # For most Docker users, package installs happen during "docker build", which # doesn't survive power loss and gets restarted clean afterwards anyhow, so @@ -122,7 +124,7 @@ EOF # https://github.com/debuerreotype/debuerreotype/issues/41 isDebianJessie="$([ -f "$targetDir/etc/os-release" ] && source "$targetDir/etc/os-release" && [ "${ID:-}" = 'debian' ] && [ "${VERSION_ID:-}" = '8' ] && echo '1')" || : - if [ -n "$isDebianJessie" ] || [[ "$aptVersion" == 0.* ]] || "$thisDir/debuerreotype-chroot" "$targetDir" dpkg --compare-versions "$aptVersion" '<<' '1.0.9.2~'; then + if [ -n "$isDebianJessie" ] || [[ "$aptVersion" == 0.* ]] || dpkg --compare-versions "$aptVersion" '<<' '1.0.9.2~'; then cat >> "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF' # https://salsa.debian.org/apt-team/apt/commit/b0f4b486e6850c5f98520ccf19da71d0ed748ae4; released in src:apt 1.0.9.2, 2014-10-02 diff -Nru debuerreotype-0.10/scripts/debuerreotype-recalculate-epoch debuerreotype-0.14/scripts/debuerreotype-recalculate-epoch --- debuerreotype-0.10/scripts/debuerreotype-recalculate-epoch 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-recalculate-epoch 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" +source "$thisDir/.constants.sh" \ + '' \ + 'rootfs' + +eval "$dgetopt" +while true; do + flag="$1"; shift + dgetopt-case "$flag" + case "$flag" in + --) break ;; + *) eusage "unknown flag '$flag'" ;; + esac +done + +targetDir="${1:-}"; shift || eusage 'missing target-dir' +[ -n "$targetDir" ] + +# ideally this would use something like "apt-get indextargets" instead of hard-coding these particular "/var/lib/apt/lists" paths, but it doesn't include the Release files :( +# also a caution from DonKult: "the Release file might really be an InRelease file which failed signature checks" + +shopt -s nullglob +releaseFiles=( "$targetDir"/var/lib/apt/lists/*_{In,}Release ) +if [ "${#releaseFiles[@]}" -eq 0 ]; then + echo >&2 "error: no 'Release' files found at /var/lib/apt/lists in '$targetDir'" + echo >&2 " did you forget to populate 'sources.list' or run 'apt-get update' first?" + exit 1 +fi + +epoch="$( + awk -F ': ' '$1 == "Date" { printf "%s%c", $2, 0 }' "${releaseFiles[@]}" \ + | xargs -r0n1 date '+%s' --date \ + | sort -un \ + | tail -1 +)" +echo "$epoch" > "$targetDir/debuerreotype-epoch" diff -Nru debuerreotype-0.10/scripts/debuerreotype-slimify debuerreotype-0.14/scripts/debuerreotype-slimify --- debuerreotype-0.10/scripts/debuerreotype-slimify 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-slimify 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' @@ -19,6 +19,20 @@ targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] +dpkgVersion="$("$thisDir/.apt-version.sh" "$targetDir" 'dpkg')" + +if ! { [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ] && dpkg --compare-versions "$dpkgVersion" '>=' '1.15.8.6~'; }; then + # --debian-eol lenny and older do not include /etc/dpkg/dpkg.cfg.d + # path-exclude/include was added in dpkg 1.15.8: https://salsa.debian.org/dpkg-team/dpkg/-/commit/4694cd64089bc72975d8ba6fbe51339023eb2e8c + echo >&2 "note: skipping $self: dpkg version ($dpkgVersion) too old to support path-exclude" + exit +fi + +# https://github.com/debuerreotype/debuerreotype/issues/10 +shopt -s nullglob +extraSpecialDirectories=( "$targetDir"/usr/share/man/man[0-9]/ ) +shopt -u nullglob + IFS=$'\n'; set -o noglob slimExcludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-excludes" | sort -u) ) slimIncludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-includes" | sort -u) ) @@ -85,3 +99,8 @@ done } >> "$dpkgCfgFile" chmod 0644 "$dpkgCfgFile" + +# https://github.com/debuerreotype/debuerreotype/issues/10 +if [ "${#extraSpecialDirectories[@]}" -gt 0 ]; then + mkdir -p "${extraSpecialDirectories[@]}" +fi diff -Nru debuerreotype-0.10/scripts/debuerreotype-tar debuerreotype-0.14/scripts/debuerreotype-tar --- debuerreotype-0.10/scripts/debuerreotype-tar 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-tar 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ --flags 'exclude:' \ --flags 'include-dev' \ diff -Nru debuerreotype-0.10/scripts/debuerreotype-version debuerreotype-0.14/scripts/debuerreotype-version --- debuerreotype-0.10/scripts/debuerreotype-version 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/debuerreotype-version 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ '' diff -Nru debuerreotype-0.10/scripts/.dpkg-arch.sh debuerreotype-0.14/scripts/.dpkg-arch.sh --- debuerreotype-0.10/scripts/.dpkg-arch.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.dpkg-arch.sh 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ '' \ 'rootfs' @@ -19,6 +19,15 @@ targetDir="${1:-}"; shift || eusage 'missing target-dir' [ -n "$targetDir" ] +if [ -s "$targetDir/etc/debian_version" ] && debVer="$(< "$targetDir/etc/debian_version")" && [ "$debVer" = '2.1' ]; then + # must be slink, where invoking "dpkg --print-architecture" leads to: + # dpkg (subprocess): failed to exec C compiler `gcc': No such file or directory + # dpkg: subprocess gcc --print-libgcc-file-name returned error exit status 2 + echo 'i386' + # (we don't support any of "alpha", "m68k", or "sparc"; see http://archive.debian.org/debian/dists/slink/ -- if we ever do, "apt-get --version" is a good candidate for scraping: "apt 0.3.11 for i386 compiled on Aug 8 1999 10:12:36") + exit +fi + arch="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg --print-architecture)" # --debian-eol woody likes to give us "i386-none" diff -Nru debuerreotype-0.10/scripts/.fix-apt-comments.sh debuerreotype-0.14/scripts/.fix-apt-comments.sh --- debuerreotype-0.10/scripts/.fix-apt-comments.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.fix-apt-comments.sh 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [file ...]' \ '0.7.22 rootfs/etc/apt/apt.conf.d/example' diff -Nru debuerreotype-0.10/scripts/.gpgv-ignore-expiration.sh debuerreotype-0.14/scripts/.gpgv-ignore-expiration.sh --- debuerreotype-0.10/scripts/.gpgv-ignore-expiration.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.gpgv-ignore-expiration.sh 2022-03-18 19:37:10.000000000 +0000 @@ -28,7 +28,8 @@ if fd="$(_status_fd "$@")" && [ -n "$fd" ]; then # older bash (3.2, lenny) doesn't support variable file descriptors (hence "eval") # (bash: syntax error near unexpected token `$fd') - eval 'exec gpgv "$@" '"$fd"'> >(sed "s/EXPKEYSIG/GOODSIG/" >&'"$fd"')' + sedExpression='s/^\[GNUPG:\] EXPKEYSIG /[GNUPG:] GOODSIG /' + eval 'exec gpgv "$@" '"$fd"'> >(sed "$sedExpression" >&'"$fd"')' fi # no "--status-fd"? no worries! ("gpgv" without "--status-fd" doesn't seem to care about expired keys, so we don't have to either) diff -Nru debuerreotype-0.10/scripts/.slimify-excludes debuerreotype-0.14/scripts/.slimify-excludes --- debuerreotype-0.10/scripts/.slimify-excludes 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.slimify-excludes 2022-03-18 19:37:10.000000000 +0000 @@ -3,17 +3,17 @@ # https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files /usr/share/doc/* -/usr/share/groff/* /usr/share/info/* /usr/share/linda/* /usr/share/lintian/overrides/* /usr/share/locale/* /usr/share/man/* +#/usr/share/groff/* (https://github.com/debuerreotype/debuerreotype/issues/87) -# https://anonscm.debian.org/cgit/collab-maint/localepurge.git/tree/usr/share/localepurge/gen-dpkg-cfg.pl?id=b32c8d46d43be2027096f5a202ac789a637ceb39#n9 -/usr/share/locale/* -/usr/share/gnome/help/*/* +# https://salsa.debian.org/elmig-guest/localepurge/-/blob/176446028ca719d65993eb01e39d7040fbbcf12d/usr/share/localepurge/gen-dpkg-cfg.pl#L9-20 /usr/share/doc/kde/HTML/*/* +/usr/share/gnome/help/*/* +/usr/share/locale/* /usr/share/omf/*/*-*.emf # see also .slimify-includes diff -Nru debuerreotype-0.10/scripts/.slimify-includes debuerreotype-0.14/scripts/.slimify-includes --- debuerreotype-0.10/scripts/.slimify-includes 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.slimify-includes 2022-03-18 19:37:10.000000000 +0000 @@ -4,14 +4,14 @@ # https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files /usr/share/doc/*/copyright -# https://anonscm.debian.org/cgit/collab-maint/localepurge.git/tree/usr/share/localepurge/gen-dpkg-cfg.pl?id=b32c8d46d43be2027096f5a202ac789a637ceb39#n9 -/usr/share/locale/locale.alias -/usr/share/gnome/help/*/C/* +# https://salsa.debian.org/elmig-guest/localepurge/-/blob/176446028ca719d65993eb01e39d7040fbbcf12d/usr/share/localepurge/gen-dpkg-cfg.pl#L22-47 /usr/share/doc/kde/HTML/C/* -/usr/share/omf/*/*-C.emf -/usr/share/locale/languages +/usr/share/gnome/help/*/C/* /usr/share/locale/all_languages /usr/share/locale/currency/* /usr/share/locale/l10n/* +/usr/share/locale/languages +/usr/share/locale/locale.alias +/usr/share/omf/*/*-C.emf # see also .slimify-excludes diff -Nru debuerreotype-0.10/scripts/.snapshot-url.sh debuerreotype-0.14/scripts/.snapshot-url.sh --- debuerreotype-0.10/scripts/.snapshot-url.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/scripts/.snapshot-url.sh 2022-03-18 19:37:10.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" +thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")" source "$thisDir/.constants.sh" \ ' [archive]' \ '2017-05-08T00:00:00Z debian-security' diff -Nru debuerreotype-0.10/steamos.sh debuerreotype-0.14/steamos.sh --- debuerreotype-0.10/steamos.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/steamos.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,188 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" -source "$thisDir/scripts/.constants.sh" \ - --flags 'no-build' \ - -- \ - '[--no-build] [suite]' \ - 'output' - -eval "$dgetopt" -build=1 -while true; do - flag="$1"; shift - dgetopt-case "$flag" - case "$flag" in - --no-build) build= ;; # for skipping "docker build" - --) break ;; - *) eusage "unknown flag '$flag'" ;; - esac -done - -outputDir="${1:-}"; shift || eusage 'missing output-dir' -suite="${1:-brewmaster}" # http://repo.steampowered.com/steamos/dists/ - -mkdir -p "$outputDir" -outputDir="$(readlink -f "$outputDir")" - -securityArgs=( - --cap-add SYS_ADMIN - --cap-drop SETFCAP -) -if docker info | grep -q apparmor; then - # AppArmor blocks mount :) - securityArgs+=( - --security-opt apparmor=unconfined - ) -fi - -ver="$("$thisDir/scripts/debuerreotype-version")" -ver="${ver%% *}" -dockerImage="debuerreotype/debuerreotype:$ver" -[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" - -steamDockerImage="$dockerImage-steamos" -[ -z "$build" ] || docker build -t "$steamDockerImage" - <<-EODF - FROM $dockerImage - # http://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/?C=M;O=D - RUN wget -O valve.deb 'http://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/valve-archive-keyring_0.5+bsos3_all.deb' \\ - && apt install -y ./valve.deb \\ - && rm valve.deb -EODF - -docker run \ - --rm \ - "${securityArgs[@]}" \ - --tmpfs /tmp:dev,exec,suid,noatime \ - -w /tmp \ - -e suite="$suite" \ - -e TZ='UTC' -e LC_ALL='C' \ - "$steamDockerImage" \ - bash -Eeuo pipefail -c ' - set -x - - mirror="http://repo.steampowered.com/steamos" - - dpkgArch="$(dpkg --print-architecture | awk -F- "{ print \$NF }")" - - exportDir="output" - outputDir="$exportDir/steamos/$dpkgArch/$suite" - - debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" - - mkdir -p "$outputDir" - wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" - wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" - gpgv \ - --keyring /usr/share/keyrings/valve-archive-keyring.gpg \ - "$outputDir/Release.gpg" \ - "$outputDir/Release" - - { - debuerreotype-init --non-debian \ - --debootstrap-script /usr/share/debootstrap/scripts/jessie \ - --keyring /usr/share/keyrings/valve-archive-keyring.gpg \ - --include valve-archive-keyring \ - --exclude debian-archive-keyring \ - --no-merged-usr \ - rootfs "$suite" "$mirror" - echo "deb $mirror $suite main contrib non-free" | tee rootfs/etc/apt/sources.list - - epoch="$(< rootfs/debuerreotype-epoch)" - touch_epoch() { - while [ "$#" -gt 0 ]; do - local f="$1"; shift - touch --no-dereference --date="@$epoch" "$f" - done - } - - debuerreotype-minimizing-config rootfs - debuerreotype-apt-get rootfs update -qq - debuerreotype-apt-get rootfs dist-upgrade -yqq - - # make a couple copies of rootfs so we can create other variants - for variant in slim sbuild; do - mkdir "rootfs-$variant" - tar -cC rootfs . | tar -xC "rootfs-$variant" - done - - # prefer iproute2 if it exists - iproute=iproute2 - if ! debuerreotype-chroot rootfs apt-get install -qq -s iproute2 &> /dev/null; then - # poor wheezy - iproute=iproute - fi - debuerreotype-apt-get rootfs install -y --no-install-recommends iputils-ping $iproute - - debuerreotype-slimify rootfs-slim - - # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild - # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 - # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 - fakeroot=fakeroot - if [[ "$suite" == alchemist* ]]; then - # poor alchemist - fakeroot= - fi - debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential $fakeroot - - create_artifacts() { - local targetBase="$1"; shift - local rootfs="$1"; shift - local suite="$1"; shift - local variant="$1"; shift - - if [ "$variant" != "sbuild" ]; then - debuerreotype-tar "$rootfs" "$targetBase.tar.xz" - else - # sbuild needs "deb-src" entries - debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list - - # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes - # (which are used in sbuild for "--extra-package") - # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - # ... - # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" - # TODO figure out the bug and fix it in APT instead /o\ - - # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") - # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 - debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" - fi - du -hsx "$targetBase.tar.xz" - - sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" - touch_epoch "$targetBase.tar.xz.sha256" - - debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" - echo "$epoch" > "$targetBase.debuerreotype-epoch" - touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" - - for f in debian_version os-release apt/sources.list; do - targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" - cp "$rootfs/etc/$f" "$targetFile" - touch_epoch "$targetFile" - done - } - - for rootfs in rootfs*/; do - rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... - - du -hsx "$rootfs" - - variant="${rootfs#rootfs}" # "", "-slim", ... - variant="${variant#-}" # "", "slim", ... - - variantDir="$outputDir/$variant" - mkdir -p "$variantDir" - - targetBase="$variantDir/rootfs" - - create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" - done - } >&2 - - tar -cC "$exportDir" . - ' | tar -xvC "$outputDir" diff -Nru debuerreotype-0.10/.travis.sh debuerreotype-0.14/.travis.sh --- debuerreotype-0.10/.travis.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/.travis.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -epoch="$(TZ=UTC date --date "$TIMESTAMP" +%s)" -serial="$(TZ=UTC date --date "@$epoch" +%Y%m%d)" - -buildArgs=() -if [ "$SUITE" = 'eol' ]; then - buildArgs+=( '--eol' ) - SUITE="$CODENAME" -elif [ -n "${CODENAME:-}" ]; then - buildArgs+=( '--codename-copy' ) -fi -if [ -n "${ARCH:-}" ]; then - buildArgs+=( "--arch=${ARCH}" ) - if [ "$ARCH" != 'i386' ]; then - buildArgs+=( '--qemu' ) - fi -fi -buildArgs+=( travis "$SUITE" "@$epoch" ) - -checkFile="travis/$serial/${ARCH:-amd64}/${CODENAME:-$SUITE}/rootfs.tar.xz" - -set -x - -./scripts/debuerreotype-version -./build.sh "${buildArgs[@]}" - -real="$(sha256sum "$checkFile" | cut -d' ' -f1)" -[ -z "$SHA256" ] || [ "$SHA256" = "$real" ] diff -Nru debuerreotype-0.10/.travis.yml debuerreotype-0.14/.travis.yml --- debuerreotype-0.10/.travis.yml 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,40 +0,0 @@ -language: bash -services: docker - -env: - - SUITE=stable CODENAME=jessie TIMESTAMP=2017-01-01T00:00:00Z SHA256=55ba54fdca819df18d813be36503b0a02abf1570c3bf5999b10891ccca5448e2 - - SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=55ba54fdca819df18d813be36503b0a02abf1570c3bf5999b10891ccca5448e2 - - SUITE=testing CODENAME=stretch TIMESTAMP=2017-01-01T00:00:00Z SHA256=1608c820c1d9c9d8adf210f80b1d751e5c26179aa27a1c1ddb8e41ae0222d8c4 - - SUITE=stretch CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=1608c820c1d9c9d8adf210f80b1d751e5c26179aa27a1c1ddb8e41ae0222d8c4 - - SUITE=unstable CODENAME=sid TIMESTAMP=2017-01-01T00:00:00Z SHA256=49a5152822ec9f0e1a61ff1d02671681f12fc1aba083f39e972f6ff897b69c80 - - SUITE=sid CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=49a5152822ec9f0e1a61ff1d02671681f12fc1aba083f39e972f6ff897b69c80 - - SUITE=oldstable CODENAME=wheezy TIMESTAMP=2017-01-01T00:00:00Z SHA256=f1bd72548e3c25ce222fb9e2bb57a5b6d4b01042180894fb05d83a0251e6dab1 - - SUITE=wheezy CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=f1bd72548e3c25ce222fb9e2bb57a5b6d4b01042180894fb05d83a0251e6dab1 - # EOL suites testing - - SUITE=eol CODENAME=etch TIMESTAMP=2017-01-01T00:00:00Z SHA256=b48e999ab4fda1720b0dc863d38cdd4d6b55530f34f262a28949eb6173102da9 - - SUITE=eol CODENAME=lenny TIMESTAMP=2017-01-01T00:00:00Z SHA256=1a2fffd34daa4a6bb968aebe86480a4093035a23700ec5f2e883423b9b4dcfa7 - - SUITE=eol CODENAME=woody ARCH=i386 TIMESTAMP=2017-01-01T00:00:00Z SHA256=ef4bc81e31db51fa9f095811ddbcc8a005f05f098596317d5a138fa90157bf40 - # qemu-debootstrap testing - - ARCH=arm64 SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=893efc1b9db1ba2df4f171d4422194a408f9810d3b55d9b0cd66fcc7722f7567 - # a few entries for "today" to try and catch issues like https://github.com/debuerreotype/debuerreotype/issues/41 sooner - - SUITE=unstable CODENAME= TIMESTAMP="today 00:00:00" SHA256= - - SUITE=stable CODENAME= TIMESTAMP="today 00:00:00" SHA256= - - SUITE=oldstable CODENAME= TIMESTAMP="today 00:00:00" SHA256= - -addons: - apt: - packages: - - binfmt-support - - qemu-user-static - -before_script: - - docker run -d --name squignix --restart always tianon/squignix # TODO temporary!! (once https://github.com/tianon/pgp-happy-eyeballs/tree/travis-squignix is deleted, this should be) -- squignix is necessary for building etch and woody who otherwise are so poorly behaved they get rate limited by snapshot.d.o (https://travis-ci.org/debuerreotype/debuerreotype/builds/479633791) - - wget -qO- https://github.com/tianon/pgp-happy-eyeballs/raw/713f2a81bf3eac1752f0c41b271444f5b57e93c9/hack-my-builds.sh | bash - -script: - - travis_retry ./.travis.sh - -after_script: - - docker images - - docker logs rawdns - - docker logs squignix # TODO temporary!! (see above) diff -Nru debuerreotype-0.10/ubuntu.sh debuerreotype-0.14/ubuntu.sh --- debuerreotype-0.10/ubuntu.sh 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/ubuntu.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" -source "$thisDir/scripts/.constants.sh" \ - --flags 'no-build' \ - --flags 'arch:' \ - -- \ - '[--no-build] [--arch=] ' \ - 'output xenial ---arch arm64 output bionic' - -eval "$dgetopt" -build=1 -arch= -while true; do - flag="$1"; shift - dgetopt-case "$flag" - case "$flag" in - --no-build) build= ;; # for skipping "docker build" - --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init - --) break ;; - *) eusage "unknown flag '$flag'" ;; - esac -done - -outputDir="${1:-}"; shift || eusage 'missing output-dir' -suite="${1:-}"; shift || eusage 'missing suite' - -mkdir -p "$outputDir" -outputDir="$(readlink -f "$outputDir")" - -securityArgs=( - --cap-add SYS_ADMIN - --cap-drop SETFCAP -) -if docker info | grep -q apparmor; then - # AppArmor blocks mount :) - securityArgs+=( - --security-opt apparmor=unconfined - ) -fi - -ver="$("$thisDir/scripts/debuerreotype-version")" -ver="${ver%% *}" -dockerImage="debuerreotype/debuerreotype:$ver" -[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir" - -ubuntuDockerImage="$dockerImage-ubuntu" -[ -z "$build" ] || docker build -t "$ubuntuDockerImage" - <<-EODF - FROM $dockerImage - RUN apt-get update \\ - && apt-get install -y --no-install-recommends ubuntu-archive-keyring \\ - && rm -rf /var/lib/apt/lists/* -EODF - -docker run \ - --rm \ - "${securityArgs[@]}" \ - -v /tmp \ - -w /tmp \ - -e suite="$suite" \ - -e arch="$arch" \ - -e TZ='UTC' -e LC_ALL='C' \ - "$ubuntuDockerImage" \ - bash -Eeuo pipefail -c ' - set -x - - dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- "{ print \$NF }")}" - - case "$dpkgArch" in - amd64|i386) - mirror="http://archive.ubuntu.com/ubuntu" - secmirror="http://security.ubuntu.com/ubuntu" - ;; - *) - mirror="http://ports.ubuntu.com/ubuntu-ports" - secmirror="$mirror" # no separate security mirror for ports - ;; - esac - - exportDir="output" - outputDir="$exportDir/ubuntu/$dpkgArch/$suite" - - debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")" - - mkdir -p "$outputDir" - wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" - wget -O "$outputDir/Release" "$mirror/dists/$suite/Release" - gpgv \ - --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \ - "$outputDir/Release.gpg" \ - "$outputDir/Release" - - { - debuerreotype-init --non-debian \ - --arch="$dpkgArch" \ - --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \ - --no-merged-usr \ - rootfs "$suite" "$mirror" - # TODO setup proper sources.list for Ubuntu - # deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse - # deb http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse - # deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse - # deb http://security.ubuntu.com/ubuntu xenial-security main restricted universe multiverse - - epoch="$(< rootfs/debuerreotype-epoch)" - touch_epoch() { - while [ "$#" -gt 0 ]; do - local f="$1"; shift - touch --no-dereference --date="@$epoch" "$f" - done - } - - debuerreotype-minimizing-config rootfs - debuerreotype-apt-get rootfs update -qq - debuerreotype-apt-get rootfs dist-upgrade -yqq - - # make a couple copies of rootfs so we can create other variants - for variant in slim sbuild; do - mkdir "rootfs-$variant" - tar -cC rootfs . | tar -xC "rootfs-$variant" - done - - debuerreotype-apt-get rootfs install -y --no-install-recommends iproute2 iputils-ping - - debuerreotype-slimify rootfs-slim - - # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild - # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26 - # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194 - debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot - - create_artifacts() { - local targetBase="$1"; shift - local rootfs="$1"; shift - local suite="$1"; shift - local variant="$1"; shift - - if [ "$variant" != "sbuild" ]; then - debuerreotype-tar "$rootfs" "$targetBase.tar.xz" - else - # sbuild needs "deb-src" entries - debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list - - # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes - # (which are used in sbuild for "--extra-package") - # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - # ... - # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes" - # TODO figure out the bug and fix it in APT instead /o\ - - # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar") - # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521 - debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz" - fi - du -hsx "$targetBase.tar.xz" - - sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256" - touch_epoch "$targetBase.tar.xz.sha256" - - debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest" - echo "$epoch" > "$targetBase.debuerreotype-epoch" - touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" - - for f in debian_version os-release apt/sources.list; do - targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")" - cp "$rootfs/etc/$f" "$targetFile" - touch_epoch "$targetFile" - done - } - - for rootfs in rootfs*/; do - rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ... - - du -hsx "$rootfs" - - variant="${rootfs#rootfs}" # "", "-slim", ... - variant="${variant#-}" # "", "slim", ... - - variantDir="$outputDir/$variant" - mkdir -p "$variantDir" - - targetBase="$variantDir/rootfs" - - create_artifacts "$targetBase" "$rootfs" "$suite" "$variant" - done - } >&2 - - tar -cC "$exportDir" . - ' | tar -xvC "$outputDir" diff -Nru debuerreotype-0.10/.validate-debian.sh debuerreotype-0.14/.validate-debian.sh --- debuerreotype-0.10/.validate-debian.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/.validate-debian.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +epoch="$(TZ=UTC date --date "$TIMESTAMP" +%s)" +serial="$(TZ=UTC date --date "@$epoch" +%Y%m%d)" + +buildArgs=() +if [ "$SUITE" = 'eol' ]; then + buildArgs+=( '--eol' ) + SUITE="$CODENAME" +elif [ -n "${CODENAME:-}" ]; then + buildArgs+=( '--codename-copy' ) +fi +if [ -n "${ARCH:-}" ]; then + buildArgs+=( "--arch=${ARCH}" ) + if [ "$ARCH" != 'i386' ]; then + if [ "$ARCH" != 'arm64' ]; then + buildArgs+=( '--ports' ) + fi + fi +fi +buildArgs+=( validate "$SUITE" "@$epoch" ) + +checkFile="validate/$serial/${ARCH:-amd64}/${CODENAME:-$SUITE}/rootfs.tar.xz" +mkdir -p validate + +set -x + +./scripts/debuerreotype-version +./docker-run.sh --pull ./examples/debian.sh "${buildArgs[@]}" + +real="$(sha256sum "$checkFile" | cut -d' ' -f1)" +[ -z "$SHA256" ] || [ "$SHA256" = "$real" ] diff -Nru debuerreotype-0.10/.validate-ubuntu.sh debuerreotype-0.14/.validate-ubuntu.sh --- debuerreotype-0.10/.validate-ubuntu.sh 1970-01-01 00:00:00.000000000 +0000 +++ debuerreotype-0.14/.validate-ubuntu.sh 2022-03-18 19:37:10.000000000 +0000 @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +dockerImage="$(./.docker-image.sh)" +dockerImage+='-ubuntu' +{ + cat Dockerfile - <<-'EODF' + RUN set -eux; \ +# https://bugs.debian.org/929165 :( +# http://snapshot.debian.org/package/ubuntu-keyring/ +# http://snapshot.debian.org/package/ubuntu-keyring/2020.06.17.1-1/ + wget -O ubuntu-keyring.deb 'http://snapshot.debian.org/archive/debian/20210307T083530Z/pool/main/u/ubuntu-keyring/ubuntu-keyring_2020.06.17.1-1_all.deb'; \ + echo 'c2d8c4a9be6244bbea80c2e0e7624cbd3a2006a2 *ubuntu-keyring.deb' | sha1sum --strict --check -; \ + apt-get install -y --no-install-recommends ./ubuntu-keyring.deb; \ + rm ubuntu-keyring.deb + EODF +} | docker build --pull --tag "$dockerImage" --file - . + +mkdir -p validate + +set -x + +./scripts/debuerreotype-version +./docker-run.sh --image="$dockerImage" --no-build ./examples/ubuntu.sh validate "$SUITE" diff -Nru debuerreotype-0.10/VERSION debuerreotype-0.14/VERSION --- debuerreotype-0.10/VERSION 2019-04-23 20:34:39.000000000 +0000 +++ debuerreotype-0.14/VERSION 2022-03-18 19:37:10.000000000 +0000 @@ -1 +1 @@ -0.10 +0.14