diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/data/92-unattended-upgrades unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/data/92-unattended-upgrades --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/data/92-unattended-upgrades 1970-01-01 00:00:00.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/data/92-unattended-upgrades 2019-10-18 11:29:41.000000000 +0000 @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ -x /usr/share/unattended-upgrades/update-motd-unattended-upgrades ]; then + exec /usr/share/unattended-upgrades/update-motd-unattended-upgrades +fi diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/data/update-motd-unattended-upgrades unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/data/update-motd-unattended-upgrades --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/data/update-motd-unattended-upgrades 1970-01-01 00:00:00.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/data/update-motd-unattended-upgrades 2019-10-18 11:29:41.000000000 +0000 @@ -0,0 +1,11 @@ +#!/bin/sh +# +# helper for update-motd + +if [ -f /var/lib/unattended-upgrades/kept-back ]; then + cat < Fri, 18 Oct 2019 13:29:41 +0200 + unattended-upgrades (1.1ubuntu1.18.04.7~16.04.3) xenial; urgency=medium * Detect changes to moved conffiles (LP: #1823872) diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/debian/dirs unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/debian/dirs --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/debian/dirs 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/debian/dirs 2019-10-18 11:29:41.000000000 +0000 @@ -2,4 +2,5 @@ var/log/unattended-upgrades etc/logrotate.d etc/apt/apt.conf.d -etc/kernel/postinst.d \ No newline at end of file +etc/kernel/postinst.d +var/lib/unattended-upgrades diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/debian/tests/control unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/debian/tests/control --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/debian/tests/control 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/debian/tests/control 2019-10-18 11:29:41.000000000 +0000 @@ -8,4 +8,4 @@ Tests: upgrade-between-snapshots Depends: @, @builddeps@, debootstrap -Restrictions: needs-root, build-needed +Restrictions: needs-root, build-needed, flaky diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/debian/tests/upgrade-all-security unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/debian/tests/upgrade-all-security --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/debian/tests/upgrade-all-security 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/debian/tests/upgrade-all-security 2019-10-18 11:29:41.000000000 +0000 @@ -51,6 +51,15 @@ chroot_exec "$chroot_dir" apt-get update +# test update-motd when it is available +if chroot_exec "$chroot_dir" apt-cache show update-motd > /dev/null 2>&1; then + chroot_exec "$chroot_dir" apt-get -y install update-motd + echo "Checking motd snippet of unattended-upgrades..." + echo "fake-foo libfoo1" > "$chroot_dir"/var/lib/unattended-upgrades/kept-back + chroot_exec "$chroot_dir" update-motd + grep -q "2 updates could not be installed automatically" "$chroot_dir"/run/motd || (echo "Motd does not show packgages kept back! Exiting..." && exit 1) +fi + # save list of manually installed packages chroot_exec "$chroot_dir" apt-mark showmanual > "$chroot_dir/tmp/manual" @@ -64,3 +73,5 @@ ! grep "/$distro-security " "$chroot_dir/tmp/updates-left" || (echo "Security upgrades are held back! Exiting..." && exit 1) +echo "Checking if /var/lib/unattended-upgrades/kept-back was removed." +! [ -f "$chroot_dir/var/lib/unattended-upgrades/kept-back" ] || (echo "kept-back file still exists! Exiting..." && exit 1) diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/setup.py unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/setup.py --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/setup.py 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/setup.py 2019-10-18 11:29:41.000000000 +0000 @@ -15,11 +15,14 @@ data_files=[ ('../etc/logrotate.d/', ["data/logrotate.d/unattended-upgrades"]), + ('../etc/update-motd.d/', + ["data/92-unattended-upgrades"]), ('../usr/share/unattended-upgrades/', ["data/20auto-upgrades", "data/20auto-upgrades-disabled", "data/50unattended-upgrades", - "unattended-upgrade-shutdown"]), + "unattended-upgrade-shutdown", + "data/update-motd-unattended-upgrades"]), ('../usr/share/man/man8/', ["man/unattended-upgrade.8"]), ('../etc/pm/sleep.d/', diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_blacklisted_wrong_origin.py unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_blacklisted_wrong_origin.py --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_blacklisted_wrong_origin.py 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_blacklisted_wrong_origin.py 2019-10-18 11:29:41.000000000 +0000 @@ -38,7 +38,7 @@ []) self.assertListEqual([], pkgs_to_upgrade) - self.assertListEqual([], pkgs_kept_back) + self.assertEqual(0, len(pkgs_kept_back)) if __name__ == "__main__": diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_mail.py unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_mail.py --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_mail.py 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_mail.py 2019-10-18 11:29:41.000000000 +0000 @@ -85,7 +85,7 @@ """ return input tuple for send_summary_mail """ pkgs = "\n".join(["2vcard"]) res = successful - pkgs_kept_back = ["linux-image"] + pkgs_kept_back = {"Debian wheezy-security": ["linux-image"]} pkgs_removed = ["telnet"] pkgs_kept_installed = ["hello"] # include some unicode chars here for good measure @@ -167,7 +167,8 @@ self.assertTrue("[package on hold]" in mail_txt) self._verify_common_mail_content(mail_txt) self.assertTrue( - "Packages with upgradable origin but kept back:\n linux-image" + "Packages with upgradable origin but kept back:\n" + " Debian wheezy-security:\n linux-image" in mail_txt) def test_summary_mail_blacklisted_only(self): @@ -185,7 +186,8 @@ self.assertTrue("[package on hold]" in mail_txt) self._verify_common_mail_content(mail_txt) self.assertTrue( - "Packages with upgradable origin but kept back:\n linux-image" + "Packages with upgradable origin but kept back:\n" + " Debian wheezy-security:\n linux-image" in mail_txt) self.assertFalse( "Packages that attempted to upgrade:\n 2vcard" in mail_txt) diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_motd.py unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_motd.py --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_motd.py 1970-01-01 00:00:00.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_motd.py 2019-10-18 11:29:41.000000000 +0000 @@ -0,0 +1,35 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +import logging +import os +import shutil +import tempfile +import unittest + +from unattended_upgrade import update_kept_packages + + +class MotdTestCase(unittest.TestCase): + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmpdir) + self.addCleanup(logging.shutdown) + + def test_packages_kept(self): + pkgs_kept_back = {"Debian wheezy-security": ["linux-image"], + "Debian wheezy": ["hello", "tworld"]} + update_kept_packages(pkgs_kept_back, os.path.join(self.tmpdir, + "kept-back")) + with open(os.path.join(self.tmpdir, "kept-back"), "rb") as fp: + kept_txt = fp.read().decode("utf-8") + self.assertEqual('hello linux-image tworld', kept_txt) + update_kept_packages({}, os.path.join(self.tmpdir, "kept-back")) + self.assertFalse( + os.path.exists(os.path.join(self.tmpdir, "kept-back"))) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + unittest.main() diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_origin_pattern.py unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_origin_pattern.py --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_origin_pattern.py 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_origin_pattern.py 2019-10-18 11:29:41.000000000 +0000 @@ -9,7 +9,7 @@ import unattended_upgrade from unattended_upgrade import ( check_changes_for_sanity, - is_allowed_origin, + is_in_allowed_origin, get_distro_codename, match_whitelist_string, UnknownMatcherError, @@ -100,7 +100,7 @@ self.assertTrue("o=MoreCorp\, eink,a=stable" in allowed_origins) # test whitelist pkg = self._get_mock_package() - self.assertTrue(is_allowed_origin(pkg.candidate, allowed_origins)) + self.assertTrue(is_in_allowed_origin(pkg.candidate, allowed_origins)) def test_escaped_colon(self): apt_pkg.config.clear("Unattended-Upgrade") diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_rewind.py unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_rewind.py --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/test_rewind.py 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/test_rewind.py 2019-10-18 11:29:41.000000000 +0000 @@ -43,7 +43,7 @@ self.assertEqual(to_upgrade, [self.cache[p] for p in ["test-package", "test2-package", "test3-package"]]) - self.assertEqual(kept_back, ["z-package"]) + self.assertEqual(kept_back["Ubuntu lucid-security"], {"z-package"}) unattended_upgrade.rewind_cache(self.cache, to_upgrade) self.assertEqual(self.cache['test-package'].candidate.version, "2.0") diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/unattended_upgrade.py unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/unattended_upgrade.py --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/test/unattended_upgrade.py 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/test/unattended_upgrade.py 2019-10-18 11:29:41.000000000 +0000 @@ -44,8 +44,10 @@ import sys try: - from typing import AbstractSet, Dict, Iterable, List, Tuple, Union + from typing import AbstractSet, DefaultDict, Dict, Iterable, List, Tuple + from typing import Union AbstractSet # pyflakes + DefaultDict # pyflakes Dict # pyflakes Iterable # pyflakes List # pyflakes @@ -54,6 +56,7 @@ except ImportError: pass +from collections import defaultdict from datetime import date from email.message import Message from gettext import gettext as _ @@ -74,6 +77,7 @@ # the reboot required flag file used by packages REBOOT_REQUIRED_FILE = "/var/run/reboot-required" +KEPT_PACKAGES_FILE = "var/lib/unattended-upgrades/kept-back" MAIL_BINARY = "/usr/bin/mail" SENDMAIL_BINARY = "/usr/sbin/sendmail" USERS = "/usr/bin/users" @@ -229,8 +233,8 @@ for marked_pkg in self.get_changes(): if marked_pkg.name in self._cached_candidate_pkgnames: continue - if not is_allowed_origin(marked_pkg.candidate, - self.allowed_origins): + if not is_in_allowed_origin(marked_pkg.candidate, + self.allowed_origins): try: ver_in_allowed_origin(marked_pkg, self.allowed_origins) @@ -411,6 +415,38 @@ apt_pkg.pkgsystem_lock() +class KeptPkgs(defaultdict): + """ + Packages to keep by highest allowed pretty-printed origin + + """ + def add(self, pkg, cache): + # type: (apt.Package, UnattendedUpgradesCache) -> None + allowed_origins = cache.allowed_origins + for origin in ver_in_allowed_origin(pkg, allowed_origins).origins: + if is_allowed_origin(origin, allowed_origins): + self[origin.origin + " " + origin.archive].add(pkg.name) + return + + def pop_upgradable(self, cache): + # type: (UnattendedUpgradesCache) -> List[apt.Package] + upgradable = [] + empty_sets = set() + for name, pkg_set in self.items(): + remove_from_set = set() + for pkg_name in pkg_set: + pkg = cache[pkg_name] + if pkg.marked_install or pkg.marked_upgrade: + remove_from_set.add(pkg_name) + upgradable.append(pkg) + pkg_set -= remove_from_set + if not pkg_set: + empty_sets.add(name) + for empty_set in empty_sets: + del(self[empty_set]) + return upgradable + + def is_dpkg_journal_dirty(): # type: () -> bool """ @@ -723,14 +759,21 @@ return res -def is_allowed_origin(ver, allowed_origins): +def is_allowed_origin(origin, allowed_origins): + # type: (apt.package.Origin, List[str]) -> bool + for allowed in allowed_origins: + if match_whitelist_string(allowed, origin): + return True + return False + + +def is_in_allowed_origin(ver, allowed_origins): # type: (apt.package.Version, List[str]) -> bool if not ver: return False for origin in ver.origins: - for allowed in allowed_origins: - if match_whitelist_string(allowed, origin): - return True + if is_allowed_origin(origin, allowed_origins): + return True return False @@ -745,7 +788,7 @@ continue except AttributeError: pass - if is_allowed_origin(ver, allowed_origins): + if is_in_allowed_origin(ver, allowed_origins): # leave as soon as we have the highest new candidate return ver raise NoAllowedOriginError() @@ -836,7 +879,7 @@ if not any([o.trusted for o in pkg.candidate.origins]): logging.debug("pkg %s is untrusted" % pkg.name) return False - if not is_allowed_origin(pkg.candidate, allowed_origins): + if not is_in_allowed_origin(pkg.candidate, allowed_origins): logging.debug("pkg %s not in allowed origin" % pkg.name) return False if not is_pkg_change_allowed(pkg, blacklist, whitelist): @@ -1124,9 +1167,15 @@ return ret -def send_summary_mail(pkgs, res, pkgs_kept_back, pkgs_removed, - pkgs_kept_installed, mem_log, dpkg_log_content): - # type: (str, bool, List[str], List[str], List[str], StringIO, str) -> None +def send_summary_mail(pkgs, # type: str + res, # type: bool + pkgs_kept_back, # type: KeptPkgs + pkgs_removed, # type: List[str] + pkgs_kept_installed, # type: List[str] + mem_log, # type: StringIO + dpkg_log_content, # type: str + ): + # type: (...) -> None """ send mail (if configured in Unattended-Upgrade::Mail) """ to_email = apt_pkg.config.find("Unattended-Upgrade::Mail", "") if not to_email: @@ -1168,9 +1217,10 @@ body += "\n" if pkgs_kept_back: body += _("Packages with upgradable origin but kept back:\n") - body += " " + wrap(" ".join(pkgs_kept_back), 70, " ") + for origin, origin_pkgs in pkgs_kept_back.items(): + body += " " + origin + ":\n" + body += " " + wrap(" ".join(origin_pkgs), 70, " ") + "\n" body += "\n" - body += "\n" if pkgs_removed: body += _("Packages that were auto-removed:\n") body += " " + wrap(" ".join(pkgs_removed), 70, " ") @@ -1389,7 +1439,7 @@ def try_to_upgrade(pkg, # type: apt.Package pkgs_to_upgrade, # type: List[apt.Package] - pkgs_kept_back, # type: List[str] + pkgs_kept_back, # type: KeptPkgs cache, # type: apt.Cache allowed_origins, # type: List[str] blacklisted_pkgs, # type: List[str] @@ -1415,24 +1465,20 @@ # re-eval pkgs_kept_back as the resolver may fail to # directly upgrade a pkg, but that may work during # a subsequent operation, see debian bug #639840 - for pkgname in pkgs_kept_back: - if (cache[pkgname].marked_install or - cache[pkgname].marked_upgrade): - pkgs_kept_back.remove(pkgname) - pkgs_to_upgrade.append(cache[pkgname]) + pkgs_to_upgrade.extend(pkgs_kept_back.pop_upgradable(cache)) else: logging.debug("sanity check failed for: %s" % str({str(p.candidate) for p in cache.get_changes()})) rewind_cache(cache, pkgs_to_upgrade) - pkgs_kept_back.append(pkg.name) + pkgs_kept_back.add(pkg, cache) except (SystemError, NoAllowedOriginError) as e: # can't upgrade logging.warning( _("package %s upgradable but fails to " "be marked for upgrade (%s)"), pkg.name, e) rewind_cache(cache, pkgs_to_upgrade) - pkgs_kept_back.append(pkg.name) + pkgs_kept_back.add(pkg, cache) def calculate_upgradable_pkgs(cache, # type: apt.Cache @@ -1441,9 +1487,9 @@ blacklisted_pkgs, # type: List[str] whitelisted_pkgs, # type: List[str] ): - # type: (...) -> Tuple[List[apt.Package], List[str]] + # type: (...) -> Tuple[List[apt.Package], KeptPkgs] pkgs_to_upgrade = [] # type: List[apt.Package] - pkgs_kept_back = [] # type: List[str] + pkgs_kept_back = KeptPkgs(set) # now do the actual upgrade for pkg in cache: @@ -1645,10 +1691,27 @@ return False -def main(options, rootdir=""): +def update_kept_packages(kept_pkgs, kept_file): + # type: (DefaultDict[str, List[str]], str) -> None + if kept_pkgs: + pkgs_all_origins = set() + for origin_pkgs in kept_pkgs.values(): + pkgs_all_origins.update(origin_pkgs) + try: + with open(kept_file, "w") as kf: + kf.write(" ".join(sorted(pkgs_all_origins))) + except Exception: + logging.error(_("Could not open %s for saving list of packages " + "kept back." % kept_file)) + else: + if os.path.exists(kept_file): + os.remove(kept_file) + + +def main(options, rootdir="/"): # type: (Options, str) -> int # useful for testing - if rootdir: + if not rootdir == "/": _setup_alternative_rootdir(rootdir) # setup logging @@ -1829,8 +1892,10 @@ logging.warning(_("Package %s has conffile prompt and " "needs to be upgraded manually"), pkgname_from_deb(item.destfile)) - blacklisted_pkgs.append(pkgname_from_deb(item.destfile)) - pkgs_kept_back.append(pkgname_from_deb(item.destfile)) + pkg_name = pkgname_from_deb(item.destfile) + if not is_pkgname_in_blacklist(pkg_name, blacklisted_pkgs): + blacklisted_pkgs.append("%s$" % re.escape(pkg_name)) + pkgs_kept_back.add(cache[pkg_name], cache) pkg_conffile_prompt = True # redo the selection about the packages to upgrade based on the new @@ -1855,7 +1920,7 @@ pkgs_to_upgrade.append(pkg) else: if not (pkg.name in pkgs_kept_back): - pkgs_kept_back.append(pkg.name) + pkgs_kept_back.add(pkg, cache) logging.info(_("package %s not upgraded"), pkg.name) cache.clear() for pkg2 in pkgs_to_upgrade: @@ -2030,6 +2095,8 @@ if not options.dry_run: reboot_if_requested_and_needed() if successful_run: + update_kept_packages(pkgs_kept_back, + os.path.join(rootdir, KEPT_PACKAGES_FILE)) return 0 else: return 1 diff -Nru unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/unattended-upgrade unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/unattended-upgrade --- unattended-upgrades-1.1ubuntu1.18.04.7~16.04.3/unattended-upgrade 2019-04-29 10:23:14.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.7~16.04.4/unattended-upgrade 2019-10-18 11:29:41.000000000 +0000 @@ -44,8 +44,10 @@ import sys try: - from typing import AbstractSet, Dict, Iterable, List, Tuple, Union + from typing import AbstractSet, DefaultDict, Dict, Iterable, List, Tuple + from typing import Union AbstractSet # pyflakes + DefaultDict # pyflakes Dict # pyflakes Iterable # pyflakes List # pyflakes @@ -54,6 +56,7 @@ except ImportError: pass +from collections import defaultdict from datetime import date from email.message import Message from gettext import gettext as _ @@ -74,6 +77,7 @@ # the reboot required flag file used by packages REBOOT_REQUIRED_FILE = "/var/run/reboot-required" +KEPT_PACKAGES_FILE = "var/lib/unattended-upgrades/kept-back" MAIL_BINARY = "/usr/bin/mail" SENDMAIL_BINARY = "/usr/sbin/sendmail" USERS = "/usr/bin/users" @@ -229,8 +233,8 @@ for marked_pkg in self.get_changes(): if marked_pkg.name in self._cached_candidate_pkgnames: continue - if not is_allowed_origin(marked_pkg.candidate, - self.allowed_origins): + if not is_in_allowed_origin(marked_pkg.candidate, + self.allowed_origins): try: ver_in_allowed_origin(marked_pkg, self.allowed_origins) @@ -411,6 +415,38 @@ apt_pkg.pkgsystem_lock() +class KeptPkgs(defaultdict): + """ + Packages to keep by highest allowed pretty-printed origin + + """ + def add(self, pkg, cache): + # type: (apt.Package, UnattendedUpgradesCache) -> None + allowed_origins = cache.allowed_origins + for origin in ver_in_allowed_origin(pkg, allowed_origins).origins: + if is_allowed_origin(origin, allowed_origins): + self[origin.origin + " " + origin.archive].add(pkg.name) + return + + def pop_upgradable(self, cache): + # type: (UnattendedUpgradesCache) -> List[apt.Package] + upgradable = [] + empty_sets = set() + for name, pkg_set in self.items(): + remove_from_set = set() + for pkg_name in pkg_set: + pkg = cache[pkg_name] + if pkg.marked_install or pkg.marked_upgrade: + remove_from_set.add(pkg_name) + upgradable.append(pkg) + pkg_set -= remove_from_set + if not pkg_set: + empty_sets.add(name) + for empty_set in empty_sets: + del(self[empty_set]) + return upgradable + + def is_dpkg_journal_dirty(): # type: () -> bool """ @@ -723,14 +759,21 @@ return res -def is_allowed_origin(ver, allowed_origins): +def is_allowed_origin(origin, allowed_origins): + # type: (apt.package.Origin, List[str]) -> bool + for allowed in allowed_origins: + if match_whitelist_string(allowed, origin): + return True + return False + + +def is_in_allowed_origin(ver, allowed_origins): # type: (apt.package.Version, List[str]) -> bool if not ver: return False for origin in ver.origins: - for allowed in allowed_origins: - if match_whitelist_string(allowed, origin): - return True + if is_allowed_origin(origin, allowed_origins): + return True return False @@ -745,7 +788,7 @@ continue except AttributeError: pass - if is_allowed_origin(ver, allowed_origins): + if is_in_allowed_origin(ver, allowed_origins): # leave as soon as we have the highest new candidate return ver raise NoAllowedOriginError() @@ -836,7 +879,7 @@ if not any([o.trusted for o in pkg.candidate.origins]): logging.debug("pkg %s is untrusted" % pkg.name) return False - if not is_allowed_origin(pkg.candidate, allowed_origins): + if not is_in_allowed_origin(pkg.candidate, allowed_origins): logging.debug("pkg %s not in allowed origin" % pkg.name) return False if not is_pkg_change_allowed(pkg, blacklist, whitelist): @@ -1124,9 +1167,15 @@ return ret -def send_summary_mail(pkgs, res, pkgs_kept_back, pkgs_removed, - pkgs_kept_installed, mem_log, dpkg_log_content): - # type: (str, bool, List[str], List[str], List[str], StringIO, str) -> None +def send_summary_mail(pkgs, # type: str + res, # type: bool + pkgs_kept_back, # type: KeptPkgs + pkgs_removed, # type: List[str] + pkgs_kept_installed, # type: List[str] + mem_log, # type: StringIO + dpkg_log_content, # type: str + ): + # type: (...) -> None """ send mail (if configured in Unattended-Upgrade::Mail) """ to_email = apt_pkg.config.find("Unattended-Upgrade::Mail", "") if not to_email: @@ -1168,9 +1217,10 @@ body += "\n" if pkgs_kept_back: body += _("Packages with upgradable origin but kept back:\n") - body += " " + wrap(" ".join(pkgs_kept_back), 70, " ") + for origin, origin_pkgs in pkgs_kept_back.items(): + body += " " + origin + ":\n" + body += " " + wrap(" ".join(origin_pkgs), 70, " ") + "\n" body += "\n" - body += "\n" if pkgs_removed: body += _("Packages that were auto-removed:\n") body += " " + wrap(" ".join(pkgs_removed), 70, " ") @@ -1389,7 +1439,7 @@ def try_to_upgrade(pkg, # type: apt.Package pkgs_to_upgrade, # type: List[apt.Package] - pkgs_kept_back, # type: List[str] + pkgs_kept_back, # type: KeptPkgs cache, # type: apt.Cache allowed_origins, # type: List[str] blacklisted_pkgs, # type: List[str] @@ -1415,24 +1465,20 @@ # re-eval pkgs_kept_back as the resolver may fail to # directly upgrade a pkg, but that may work during # a subsequent operation, see debian bug #639840 - for pkgname in pkgs_kept_back: - if (cache[pkgname].marked_install or - cache[pkgname].marked_upgrade): - pkgs_kept_back.remove(pkgname) - pkgs_to_upgrade.append(cache[pkgname]) + pkgs_to_upgrade.extend(pkgs_kept_back.pop_upgradable(cache)) else: logging.debug("sanity check failed for: %s" % str({str(p.candidate) for p in cache.get_changes()})) rewind_cache(cache, pkgs_to_upgrade) - pkgs_kept_back.append(pkg.name) + pkgs_kept_back.add(pkg, cache) except (SystemError, NoAllowedOriginError) as e: # can't upgrade logging.warning( _("package %s upgradable but fails to " "be marked for upgrade (%s)"), pkg.name, e) rewind_cache(cache, pkgs_to_upgrade) - pkgs_kept_back.append(pkg.name) + pkgs_kept_back.add(pkg, cache) def calculate_upgradable_pkgs(cache, # type: apt.Cache @@ -1441,9 +1487,9 @@ blacklisted_pkgs, # type: List[str] whitelisted_pkgs, # type: List[str] ): - # type: (...) -> Tuple[List[apt.Package], List[str]] + # type: (...) -> Tuple[List[apt.Package], KeptPkgs] pkgs_to_upgrade = [] # type: List[apt.Package] - pkgs_kept_back = [] # type: List[str] + pkgs_kept_back = KeptPkgs(set) # now do the actual upgrade for pkg in cache: @@ -1645,10 +1691,27 @@ return False -def main(options, rootdir=""): +def update_kept_packages(kept_pkgs, kept_file): + # type: (DefaultDict[str, List[str]], str) -> None + if kept_pkgs: + pkgs_all_origins = set() + for origin_pkgs in kept_pkgs.values(): + pkgs_all_origins.update(origin_pkgs) + try: + with open(kept_file, "w") as kf: + kf.write(" ".join(sorted(pkgs_all_origins))) + except Exception: + logging.error(_("Could not open %s for saving list of packages " + "kept back." % kept_file)) + else: + if os.path.exists(kept_file): + os.remove(kept_file) + + +def main(options, rootdir="/"): # type: (Options, str) -> int # useful for testing - if rootdir: + if not rootdir == "/": _setup_alternative_rootdir(rootdir) # setup logging @@ -1829,8 +1892,10 @@ logging.warning(_("Package %s has conffile prompt and " "needs to be upgraded manually"), pkgname_from_deb(item.destfile)) - blacklisted_pkgs.append(pkgname_from_deb(item.destfile)) - pkgs_kept_back.append(pkgname_from_deb(item.destfile)) + pkg_name = pkgname_from_deb(item.destfile) + if not is_pkgname_in_blacklist(pkg_name, blacklisted_pkgs): + blacklisted_pkgs.append("%s$" % re.escape(pkg_name)) + pkgs_kept_back.add(cache[pkg_name], cache) pkg_conffile_prompt = True # redo the selection about the packages to upgrade based on the new @@ -1855,7 +1920,7 @@ pkgs_to_upgrade.append(pkg) else: if not (pkg.name in pkgs_kept_back): - pkgs_kept_back.append(pkg.name) + pkgs_kept_back.add(pkg, cache) logging.info(_("package %s not upgraded"), pkg.name) cache.clear() for pkg2 in pkgs_to_upgrade: @@ -2030,6 +2095,8 @@ if not options.dry_run: reboot_if_requested_and_needed() if successful_run: + update_kept_packages(pkgs_kept_back, + os.path.join(rootdir, KEPT_PACKAGES_FILE)) return 0 else: return 1