diff -Nru unattended-upgrades-1.1ubuntu1.18.04.5/debian/changelog unattended-upgrades-1.1ubuntu1.18.04.6/debian/changelog --- unattended-upgrades-1.1ubuntu1.18.04.5/debian/changelog 2018-07-18 11:22:36.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.6/debian/changelog 2018-10-02 17:18:02.000000000 +0000 @@ -1,3 +1,20 @@ +unattended-upgrades (1.1ubuntu1.18.04.6) bionic; urgency=medium + + * Unlock for dpkg operations with apt_pkg.pkgsystem_unlock_inner() when it is + available. Also stop running when reacquiring the lock fails. + Thanks to Julian Andres Klode for original partial patch (LP: #1789637) + * Skip rebuilding python-apt in upgrade autopkgtests. + Python-apt has a new build dependency making the rebuilding as is failing + and the reference handling issue is worked around in unattended-upgrades + already. (LP: #1781586) + * Stop trying when no adjustment could be made and adjust package candidates + only to lower versions (LP: #1785093) + * Skip already adjusted packages from being checked for readjusting. + This makes it clearer that the recursion ends and can also be a bit quicker. + (LP: #1785093) + + -- Balint Reczey Tue, 02 Oct 2018 19:18:02 +0200 + unattended-upgrades (1.1ubuntu1.18.04.5) bionic; urgency=medium * Stop updating the system when reacquiring the dpkg system lock fails. diff -Nru unattended-upgrades-1.1ubuntu1.18.04.5/debian/tests/common-functions unattended-upgrades-1.1ubuntu1.18.04.6/debian/tests/common-functions --- unattended-upgrades-1.1ubuntu1.18.04.5/debian/tests/common-functions 2018-07-18 11:22:36.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.6/debian/tests/common-functions 2018-10-02 17:18:02.000000000 +0000 @@ -37,24 +37,6 @@ trap "umount \"$chroot_dir/proc\"; umount \"$chroot_dir/dev/pts\"; rm -rf \"$chroot_dir\"" EXIT } -rebuild_python_apt() { - local chroot_dir - chroot_dir="$1" - - # save list of manually installed packages - chroot_exec "$chroot_dir" apt-mark showmanual > "$chroot_dir/tmp/manual" - - (cd "$chroot_dir"/root/ && apt-get source python-apt 2>&1) - chroot_exec "$chroot_dir" apt-get build-dep -y python-apt - chroot_exec "$chroot_dir" bash -c "cd /root/python-apt-* && env DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc && apt-get install -y ../*.deb" 2>&1 - - # remove packages installed as the side-effect of updating apt and python-apt - chroot_exec "$chroot_dir" apt-mark showmanual | comm -1 -3 "$chroot_dir/tmp/manual" - | chroot_exec "$chroot_dir" xargs apt-mark auto - chroot_exec "$chroot_dir" apt-get -y autoremove - chroot_exec "$chroot_dir" apt-get clean - rm "$chroot_dir"/tmp/manual -} - run_u_u() { local chroot_dir chroot_dir="$1" diff -Nru unattended-upgrades-1.1ubuntu1.18.04.5/debian/tests/upgrade-all-security unattended-upgrades-1.1ubuntu1.18.04.6/debian/tests/upgrade-all-security --- unattended-upgrades-1.1ubuntu1.18.04.5/debian/tests/upgrade-all-security 2018-07-18 11:22:36.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.6/debian/tests/upgrade-all-security 2018-10-02 17:18:02.000000000 +0000 @@ -35,9 +35,6 @@ # apt prints "W: APT had planned for dpkg to do more than it reported back" to stderr LP: #1647638 chroot_exec "$chroot_dir" apt-get install -y xfce4 apparmor 2>&1 -# build and install updated python-apt since the one in the snapshot has memory allocation issues -rebuild_python_apt "$chroot_dir" - # install package version just built cp ../unattended-upgrades_*.deb "$chroot_dir/tmp/" chroot_exec "$chroot_dir" bash -c 'apt install -y /tmp/unattended-upgrades_*deb' 2>&1 diff -Nru unattended-upgrades-1.1ubuntu1.18.04.5/debian/tests/upgrade-between-snapshots unattended-upgrades-1.1ubuntu1.18.04.6/debian/tests/upgrade-between-snapshots --- unattended-upgrades-1.1ubuntu1.18.04.5/debian/tests/upgrade-between-snapshots 2018-07-18 11:22:36.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.6/debian/tests/upgrade-between-snapshots 2018-10-02 17:18:02.000000000 +0000 @@ -26,9 +26,6 @@ # apt prints "W: APT had planned for dpkg to do more than it reported back" to stderr LP: #1647638 chroot_exec "$chroot_dir" apt-get install -y xfce4 2>&1 -# build and install updated python-apt since the one in the snapshot has memory allocation issues -rebuild_python_apt "$chroot_dir" - # install package version just built cp ../unattended-upgrades_*.deb "$chroot_dir/tmp/" chroot_exec "$chroot_dir" bash -c 'apt install -y /tmp/unattended-upgrades_*deb' 2>&1 diff -Nru unattended-upgrades-1.1ubuntu1.18.04.5/test/unattended_upgrade.py unattended-upgrades-1.1ubuntu1.18.04.6/test/unattended_upgrade.py --- unattended-upgrades-1.1ubuntu1.18.04.5/test/unattended_upgrade.py 2018-07-18 11:22:36.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.6/test/unattended_upgrade.py 2018-10-02 17:18:02.000000000 +0000 @@ -158,7 +158,10 @@ """ try: new_cand = ver_in_allowed_origin(pkg, self.allowed_origins) - if new_cand != pkg.candidate: + # Only adjust to lower versions to avoid flipping back and forth + # and to avoid picking a newer version, not selected by apt. + # This helps avoiding upgrades to experimental's packages. + if new_cand.version < pkg.candidate.version: logging.debug("adjusting candidate version: %s" % new_cand) pkg.candidate = new_cand return True @@ -187,6 +190,8 @@ function(pkg, **kwargs) changes = self.get_changes() for marked_pkg in changes: + if marked_pkg.name in self._cached_candidate_pkgnames: + continue if not is_allowed_origin(marked_pkg.candidate, self.allowed_origins): try: @@ -200,10 +205,13 @@ pkgs_with_no_allowed_origin.append(marked_pkg) if new_pkgs_to_adjust: + new_pkg_adjusted = False for pkg_to_adjust in new_pkgs_to_adjust: - self.adjust_candidate(pkg_to_adjust) - self._cached_candidate_pkgnames.add(pkg_to_adjust.name) - self.call_adjusted(function, pkg, **kwargs) + if self.adjust_candidate(pkg_to_adjust): + self._cached_candidate_pkgnames.add(pkg_to_adjust.name) + new_pkg_adjusted = True + if new_pkg_adjusted: + self.call_adjusted(function, pkg, **kwargs) else: if pkgs_with_no_allowed_origin: raise NoAllowedOriginError @@ -351,13 +359,22 @@ def __enter__(self): # type: () -> None try: - apt_pkg.pkgsystem_unlock() - except Exception: - pass + apt_pkg.pkgsystem_unlock_inner() + except AttributeError: + try: + apt_pkg.pkgsystem_unlock() + except Exception: + # earlier python-apt used to leak lock + logging.warning("apt_pkg.pkgsystem_unlock() failed due to not " + "holding the lock but trying to continue") + pass def __exit__(self, exc_type, exc_value, exc_tb): # type: (object, object, object) -> None - apt_pkg.pkgsystem_lock() + try: + apt_pkg.pkgsystem_lock_inner() + except AttributeError: + apt_pkg.pkgsystem_lock() def is_dpkg_journal_dirty(): @@ -535,10 +552,13 @@ iprogress = LogInstallProgress(logfile_dpkg, verbose) try: - with Unlocked(): + if hasattr(apt_pkg, "pkgsystem_lock_inner"): res = cache.commit(install_progress=iprogress) + else: + with Unlocked(): + res = cache.commit(install_progress=iprogress) cache.open() - except Exception as e: + except SystemError as e: error = e if verbose: logging.exception("Exception happened during upgrade.") @@ -1572,34 +1592,6 @@ # see debian #776752 install_start_time = datetime.datetime.now().replace(microsecond=0) - # check if the journal is dirty and if so, take emergceny action - # the alternative is to leave the system potentially unsecure until - # the user comes in and fixes - if (is_dpkg_journal_dirty() and - apt_pkg.config.find_b( - "Unattended-Upgrade::AutoFixInterruptedDpkg", True)): - # ensure the dpkg database is not already locked (LP: #754330) - admindir = os.path.dirname(apt_pkg.config.find("Dir::State::Status")) - lockfd = apt_pkg.get_lock(os.path.join(admindir, "lock"), False) - if lockfd > 0: - logging.warning( - _("Unclean dpkg state detected, trying to correct")) - print(_("Unclean dpkg state detected, trying to correct")) - env = copy.copy(os.environ) - env["DEBIAN_FRONTEND"] = "noninteractive" - try: - os.close(lockfd) - output = subprocess.check_output( - ["dpkg", "--force-confold", "--configure", "-a"], - env=env, - universal_newlines=True) - except subprocess.CalledProcessError as e: - output = e.output - logging.warning(_("dpkg --configure -a output:\n%s"), output) - else: - logging.debug("Unclean dpkg state, but locked, another package " - "manager working?") - # check and get lock try: apt_pkg.pkgsystem_lock() @@ -1609,6 +1601,28 @@ print(_("Cache lock can not be acquired, exiting")) sys.exit(1) + # check if the journal is dirty and if so, take emergceny action + # the alternative is to leave the system potentially unsecure until + # the user comes in and fixes + if is_dpkg_journal_dirty() and \ + apt_pkg.config.find_b("Unattended-Upgrade::AutoFixInterruptedDpkg", + True): + logging.warning( + _("Unclean dpkg state detected, trying to correct")) + print(_("Unclean dpkg state detected, trying to correct")) + env = copy.copy(os.environ) + env["DEBIAN_FRONTEND"] = "noninteractive" + env["DPKG_FRONTEND_LOCKED"] = "1" + try: + with Unlocked(): + output = subprocess.check_output( + ["dpkg", "--force-confold", "--configure", "-a"], + env=env, + universal_newlines=True) + except subprocess.CalledProcessError as e: + output = e.output + logging.warning(_("dpkg --configure -a output:\n%s"), output) + # get a cache try: cache = UnattendedUpgradesCache(rootdir=rootdir, diff -Nru unattended-upgrades-1.1ubuntu1.18.04.5/unattended-upgrade unattended-upgrades-1.1ubuntu1.18.04.6/unattended-upgrade --- unattended-upgrades-1.1ubuntu1.18.04.5/unattended-upgrade 2018-07-18 11:22:36.000000000 +0000 +++ unattended-upgrades-1.1ubuntu1.18.04.6/unattended-upgrade 2018-10-02 17:18:02.000000000 +0000 @@ -158,7 +158,10 @@ """ try: new_cand = ver_in_allowed_origin(pkg, self.allowed_origins) - if new_cand != pkg.candidate: + # Only adjust to lower versions to avoid flipping back and forth + # and to avoid picking a newer version, not selected by apt. + # This helps avoiding upgrades to experimental's packages. + if new_cand.version < pkg.candidate.version: logging.debug("adjusting candidate version: %s" % new_cand) pkg.candidate = new_cand return True @@ -187,6 +190,8 @@ function(pkg, **kwargs) changes = self.get_changes() for marked_pkg in changes: + if marked_pkg.name in self._cached_candidate_pkgnames: + continue if not is_allowed_origin(marked_pkg.candidate, self.allowed_origins): try: @@ -200,10 +205,13 @@ pkgs_with_no_allowed_origin.append(marked_pkg) if new_pkgs_to_adjust: + new_pkg_adjusted = False for pkg_to_adjust in new_pkgs_to_adjust: - self.adjust_candidate(pkg_to_adjust) - self._cached_candidate_pkgnames.add(pkg_to_adjust.name) - self.call_adjusted(function, pkg, **kwargs) + if self.adjust_candidate(pkg_to_adjust): + self._cached_candidate_pkgnames.add(pkg_to_adjust.name) + new_pkg_adjusted = True + if new_pkg_adjusted: + self.call_adjusted(function, pkg, **kwargs) else: if pkgs_with_no_allowed_origin: raise NoAllowedOriginError @@ -351,13 +359,22 @@ def __enter__(self): # type: () -> None try: - apt_pkg.pkgsystem_unlock() - except Exception: - pass + apt_pkg.pkgsystem_unlock_inner() + except AttributeError: + try: + apt_pkg.pkgsystem_unlock() + except Exception: + # earlier python-apt used to leak lock + logging.warning("apt_pkg.pkgsystem_unlock() failed due to not " + "holding the lock but trying to continue") + pass def __exit__(self, exc_type, exc_value, exc_tb): # type: (object, object, object) -> None - apt_pkg.pkgsystem_lock() + try: + apt_pkg.pkgsystem_lock_inner() + except AttributeError: + apt_pkg.pkgsystem_lock() def is_dpkg_journal_dirty(): @@ -535,10 +552,13 @@ iprogress = LogInstallProgress(logfile_dpkg, verbose) try: - with Unlocked(): + if hasattr(apt_pkg, "pkgsystem_lock_inner"): res = cache.commit(install_progress=iprogress) + else: + with Unlocked(): + res = cache.commit(install_progress=iprogress) cache.open() - except Exception as e: + except SystemError as e: error = e if verbose: logging.exception("Exception happened during upgrade.") @@ -1572,34 +1592,6 @@ # see debian #776752 install_start_time = datetime.datetime.now().replace(microsecond=0) - # check if the journal is dirty and if so, take emergceny action - # the alternative is to leave the system potentially unsecure until - # the user comes in and fixes - if (is_dpkg_journal_dirty() and - apt_pkg.config.find_b( - "Unattended-Upgrade::AutoFixInterruptedDpkg", True)): - # ensure the dpkg database is not already locked (LP: #754330) - admindir = os.path.dirname(apt_pkg.config.find("Dir::State::Status")) - lockfd = apt_pkg.get_lock(os.path.join(admindir, "lock"), False) - if lockfd > 0: - logging.warning( - _("Unclean dpkg state detected, trying to correct")) - print(_("Unclean dpkg state detected, trying to correct")) - env = copy.copy(os.environ) - env["DEBIAN_FRONTEND"] = "noninteractive" - try: - os.close(lockfd) - output = subprocess.check_output( - ["dpkg", "--force-confold", "--configure", "-a"], - env=env, - universal_newlines=True) - except subprocess.CalledProcessError as e: - output = e.output - logging.warning(_("dpkg --configure -a output:\n%s"), output) - else: - logging.debug("Unclean dpkg state, but locked, another package " - "manager working?") - # check and get lock try: apt_pkg.pkgsystem_lock() @@ -1609,6 +1601,28 @@ print(_("Cache lock can not be acquired, exiting")) sys.exit(1) + # check if the journal is dirty and if so, take emergceny action + # the alternative is to leave the system potentially unsecure until + # the user comes in and fixes + if is_dpkg_journal_dirty() and \ + apt_pkg.config.find_b("Unattended-Upgrade::AutoFixInterruptedDpkg", + True): + logging.warning( + _("Unclean dpkg state detected, trying to correct")) + print(_("Unclean dpkg state detected, trying to correct")) + env = copy.copy(os.environ) + env["DEBIAN_FRONTEND"] = "noninteractive" + env["DPKG_FRONTEND_LOCKED"] = "1" + try: + with Unlocked(): + output = subprocess.check_output( + ["dpkg", "--force-confold", "--configure", "-a"], + env=env, + universal_newlines=True) + except subprocess.CalledProcessError as e: + output = e.output + logging.warning(_("dpkg --configure -a output:\n%s"), output) + # get a cache try: cache = UnattendedUpgradesCache(rootdir=rootdir,