diff -Nru piuparts-0.41ubuntu2/ChangeLog piuparts-0.42ubuntu1/ChangeLog --- piuparts-0.41ubuntu2/ChangeLog 2011-10-24 17:48:05.000000000 +0000 +++ piuparts-0.42ubuntu1/ChangeLog 2012-01-03 16:41:52.000000000 +0000 @@ -1,7 +1,7 @@ 2009-12-18 Holger Levsen * This changelog is obsolete since several years. See debian/changelog - instead. + instead. 2005-11-12 Lars Wirzenius @@ -11,23 +11,23 @@ 2005-10-17 Lars Wirzenius * Version 0.12. - + * /var/log/faillog ignored by default. - + * Documented that upgrade testing to experimental doesn't work. 2005-10-14 Lars Wirzenius * Version 0.11. - + * Summary of changes: - + - Checks for missing package names on command line immediately, not after creating chroot. - + - apt-get in the chroot gets configured to allow unauthenticated repositories. - + - Tweaks to list of files ignored by default. 2005-09-15 Lars Wirzenius @@ -37,9 +37,9 @@ 2005-09-15 Lars Wirzenius * piuparts.py: When creating policy-rc.d, make it executable. - + * piuparts.docbook: Added missing "if". Thanks, Jonas Meurer. - + * run-piuparts.py: Wrote. 2005-09-09 Lars Wirzenius @@ -53,12 +53,12 @@ 2005-09-05 Lars Wirzenius - * piuparts.py, piuparts.docbook: Added -k (--keep-tmpdir) option. + * piuparts.py, piuparts.docbook: Added -k (--keep-tmpdir) option. - * piuparts.py: Added some more default ignores. + * piuparts.py: Added some more default ignores. - * piuparts.py: Remember which packages own which files before - removing packages. + * piuparts.py: Remember which packages own which files before + removing packages. 2005-08-31 Lars Wirzenius @@ -72,7 +72,7 @@ code. Also, log everything identically to stdout and log file, since the old behavior was confusing to many people, as observed on IRC and at Debconf5. This also obsoletes -v (--verbose). - + * piuparts.docbook: Document that -v (--verbose) is obsolete. 2005-08-30 Lars Wirzenius @@ -90,10 +90,10 @@ of requiring that they match everything. This is less surprising to users and they can still use ^ and $ to anchor a match to the ends, if they need it. - + * Makefile: Ignore the fdmount temp files that cause upgrades to etch/sid fail at the moments, and re-enable upgrade tests. - + * piuparts.py, piuparts.docbook: Added option -t and changed the default location for temporary files and directorys (including the chroot) to be $TMPDIR (or /tmp if not set). @@ -102,21 +102,21 @@ * Makefile: Upgrade testing from sarge to etch fails for me, because of something to do with fdutils, so I have disabled it for now. - + * piuparts.py: Only use the first mirror found in sources.list, which is anyway what the manual page claims we are doing. This should reduce the problems people have been having with piuparts creating broken source.lists in the chroot when people have more than a canonical Debian repository listed. - + * piuparts.py: Remove temporary files upon error exit. - + * piuparts.docbook: Use code names instead of stable/testing/unstable in the example. 2005-07-15 Lars Wirzenius - * Version 0.7. + * Version 0.7. 2005-07-15 Lars Wirzenius @@ -127,9 +127,9 @@ * piuparts.py: Create a /usr/sbin/policy-rc.d inside the chroot when it is created. Suggested by Frank Lichtenheld. - + * piuparts.py: Fixed a message from "FAIL:" to "PASS:". Oops. - + * Makefile: Commented out an explicit mirror definition so it should now use sources.list defaults. @@ -138,7 +138,7 @@ * piuparts.py: Suggested by Frank Lichtenheld: don't run dpkg and do other work if there are no files (cleans up log file, saves a couple of seconds). - + * piuparts.py, piuparts.docbook: Implemented and documented a limit on the size of outputs of command piuparts runs. @@ -159,118 +159,118 @@ 2005-07-05 Lars Wirzenius - * Version 0.6. + * Version 0.6. 2005-07-05 Lars Wirzenius - * piuparts.py: Bugfix: when removing packages from the chroot, - don't remove packages being depended on before the packages doing - the depending. That won't work. Any fool would know it, except me. - - * piuparts.py: Implemented saving of meta data for the upgrade - testing between Debian releases. + * piuparts.py: Bugfix: when removing packages from the chroot, + don't remove packages being depended on before the packages doing + the depending. That won't work. Any fool would know it, except me. + + * piuparts.py: Implemented saving of meta data for the upgrade + testing between Debian releases. 2005-07-04 Lars Wirzenius - * piuparts.py: Made it possible to specify components to use for - a mirror. - - * piuparts.docbook: Documented the user visible changes. + * piuparts.py: Made it possible to specify components to use for + a mirror. + + * piuparts.docbook: Documented the user visible changes. 2005-06-27 Lars Wirzenius - * piuparts.py: Remove and purge dependencies before the packages - that were installed, so that problems with postrm scripts using - non-essential packages upon purge become evident. + * piuparts.py: Remove and purge dependencies before the packages + that were installed, so that problems with postrm scripts using + non-essential packages upon purge become evident. 2005-06-27 Lars Wirzenius - * piuparts.py, piuparts.docbook: Added -n / --no-ignores option. + * piuparts.py, piuparts.docbook: Added -n / --no-ignores option. 2005-06-27 Lars Wirzenius - * piuparts.py: Made mirrors mentioned in /etc/apt/sources.list be - the default mirrors. Also, --mirror can be used multiple times - now. - - * piuparts.docbook: Documented this. - - * Makefile: log to tmp3.log instead of twice to tmp2.log. + * piuparts.py: Made mirrors mentioned in /etc/apt/sources.list be + the default mirrors. Also, --mirror can be used multiple times + now. + + * piuparts.docbook: Documented this. + + * Makefile: log to tmp3.log instead of twice to tmp2.log. 2005-06-27 Lars Wirzenius - * Makefile: Use -f with gzip so it doesn't complain when installing - new version on top of old. + * Makefile: Use -f with gzip so it doesn't complain when installing + new version on top of old. 2005-06-27 Lars Wirzenius - * piuparts.py: Added /etc/ld.so.conf to list of ignored files - and changed exim related ignore patterns (adding exim4, for - example). - + * piuparts.py: Added /etc/ld.so.conf to list of ignored files + and changed exim related ignore patterns (adding exim4, for + example). + * piuparts.py: When purging, use --purge and not --remove to dpkg. - + * piuparts.py: Report version and command line arguments at startup. - + * piuparts.py: When calling install_purge_test, report package list when command line args are package names, instead of empty list. 2005-06-23 Lars Wirzenius - * Version 0.5. + * Version 0.5. 2005-06-23 Lars Wirzenius - * piuparts.py: Check symlink targets as well. + * piuparts.py: Check symlink targets as well. 2005-06-23 Lars Wirzenius - * piuparts.docbook: Added --ignore-regexp to manual page. + * piuparts.docbook: Added --ignore-regexp to manual page. 2005-06-23 Lars Wirzenius - * TODO: Added. - - * Makefile: Added test case for upgrading between distros. - - * piuparts.docbook: Rewrote DESCRIPTION and EXAMPLES sections. - - * piuparts.py: Added /var/cache/man/index.db and - /var/log/dpkg.log to ignored files. Added regexps for more - versatile ignoring and -I option. - - * piuparts.py: Disabled comparison of mtimes of files for now. - - * piuparts.py: File listings are now sorted. - - * piuparts.py: Added upgrade tests between Debian releases. + * TODO: Added. + + * Makefile: Added test case for upgrading between distros. + + * piuparts.docbook: Rewrote DESCRIPTION and EXAMPLES sections. + + * piuparts.py: Added /var/cache/man/index.db and + /var/log/dpkg.log to ignored files. Added regexps for more + versatile ignoring and -I option. + + * piuparts.py: Disabled comparison of mtimes of files for now. + + * piuparts.py: File listings are now sorted. + + * piuparts.py: Added upgrade tests between Debian releases. 2005-06-20 Lars Wirzenius - * piuparts.py: Refactored things more than a bit in preparation - for new functionality. + * piuparts.py: Refactored things more than a bit in preparation + for new functionality. 2005-06-19 Lars Wirzenius - * Version 0.4. + * Version 0.4. 2005-06-19 Lars Wirzenius - * piuparts.docbook, piuparts.py: Added -p option. - - * piuparts.docbook: Updated a description of what the program - does. + * piuparts.docbook, piuparts.py: Added -p option. + + * piuparts.docbook: Updated a description of what the program + does. 2005-06-19 Lars Wirzenius - * piuparts.docbook: Documented upgrade testing. + * piuparts.docbook: Documented upgrade testing. 2005-06-19 Lars Wirzenius - * piuparts.py: Added a simple form of upgrade testing. + * piuparts.py: Added a simple form of upgrade testing. 2005-06-19 Lars Wirzenius @@ -278,44 +278,44 @@ do package purginging in two steps: first remove, then purge. This helps find postrm scripts that use things they aren't allowed to use. - - * README: Added naming credit to Tollef Fog Heen. + + * README: Added naming credit to Tollef Fog Heen. 2005-06-18 Lars Wirzenius - * Version 0.3. + * Version 0.3. 2005-06-18 Lars Wirzenius * piuparts.py: Added /var/cache/debconf and /var/cache/debconf/passwords.dat to the list of filenames that are automatically ignored. - + * piuparts.py: After unpacking a base tarball, run apt-get update and clean on it, to be sure everything is up to date. 2005-06-18 Lars Wirzenius - * piuparts.py: Added option -a (--apt) that lets the user install - the packages to be tested via apt-get, instead of by specifying - package files. - - * piuparts.docbook: Documented -a (--apt). + * piuparts.py: Added option -a (--apt) that lets the user install + the packages to be tested via apt-get, instead of by specifying + package files. + + * piuparts.docbook: Documented -a (--apt). 2005-06-17 Lars Wirzenius - * piuparts.docbook: Added a note that piuparts is meant for people - making Debian packages before they upload them. + * piuparts.docbook: Added a note that piuparts is meant for people + making Debian packages before they upload them. 2005-06-17 Lars Wirzenius - * piuparts.py, piuparts.docbook: Added -V (--version) option. + * piuparts.py, piuparts.docbook: Added -V (--version) option. 2005-06-17 Lars Wirzenius - * piuparts.py, README: Removed notice about python-apt / apt_pkg - since it wasn't actually used. - + * piuparts.py, README: Removed notice about python-apt / apt_pkg + since it wasn't actually used. + * piuparts.py: Changed things so that we us os.walk and os.stat to find filesystem object meta data, instead of creating a tarball and then scanning that. This makes things quite a bit diff -Nru piuparts-0.41ubuntu2/debian/changelog piuparts-0.42ubuntu1/debian/changelog --- piuparts-0.41ubuntu2/debian/changelog 2011-12-31 02:06:46.000000000 +0000 +++ piuparts-0.42ubuntu1/debian/changelog 2012-01-04 10:10:57.000000000 +0000 @@ -1,3 +1,162 @@ +piuparts (0.42ubuntu1) precise; urgency=low + + * Merge from Debian testing. Remaining changes: + - In Ubuntu, debfoster tries to remove packages with priority required + even if MaxPriority=required is used. Setting MaxPriority to important + reduces the number of packages to be removed as well as false positives + generated by unwanted package removals. This can be dropped if a + proper fix for debfoster is found. + - piuparts.py: + + Set default keyring name to ubuntu-archive-keyring.pgp + + Set default distribution to precise. + + -- Mahyuddin Susanto Wed, 04 Jan 2012 17:10:09 +0700 + +piuparts (0.42) unstable; urgency=low + + [ Holger Levsen ] + * piuparts.py: + - add to self.ignored_files: /etc/blkid.tab (Closes: #638831) + - add to self.ignored_patterns: /var/lib/apt/lists/.* + - apply patch by Gregor Herrmann to fix --minimize. (Closes: #648423) + * Remove Debian.NEWS entry about source in git. (Closes: #640121) + * piuparts.py, piuparts-report.py, ChangeLog: Expand tabs to spaces. + * Remove whitespaces from whitespace-only lines. + * piuparts-report: + - create maintainer subdirs if they don't exist. + - detect tests being terminated due to excessive output. + * Add git to Build-Depends-Indep: as git describe is used to generate + part of the version string for development builds. + * Add debian/.gitignore + + [ Mika Pflüger ] + * piuparts-analyze.py: + - Rewrite to use python-debianbts to analyze if bugs are filed already. + - The BTS only tracks source versions, so remove binNMU-part from + package versions when comparing with versions from the BTS. + - Reduce noise in the output by only printing one action/advise per + package. + - Fix extraction of package versions from bug reports. Thanks to + Andreas Beckmann for catching and solving the error. + * debian/control: Add python-apt and python-debianbts to piuparts depends. + + [ Scott Schaefer ] + * debian/copyright: Make it compliant with DEP-5. + * piuparts-slave.py: + - Replace deprecated os.popen2 with subprocess.Popen. (Closes: #640646) + - Add some more logging. + - Kill children (hard-coded value, 45 minutes) to ensure test doesn't + run "forever" (Closes: #640647, #387428) + * piuparty.py: + - Kill leftover processes (via SIGTERM, then if that fails, via SIGKILL). + (Closes: #522918) + - Test for incorrect diversion handling: (Closes: #588313) + a) Existing diversions removed/modified, and/or + b) Installed diversions not removed by purge. + * piupartslib/packagesdb.py: Modify several functions in PackagesDB class + to use python 'set' type, instead of a list. This permitted replacing + inefficient combination of unique() function and random.shuffle() with + python set.pop() method. Since python prohibits storing non-hashable + object in a set, minor modifications to piuparts-report and to + piuparts-master required. (Closes: #640648) + + [ Andreas Beckmann ] + * *.py: Add vim modeline. + * piuparts.py: + - Add unittests for misbehaving symlinks. + - Fix resolving absolute symlinks of intermediate directory components, + i.e. /var/run -> /run while checking /etc/motd -> /var/run/motd. + Solves about 30000 false positives of + 'Broken symlinks: /etc/motd -> /var/run/motd'. (Closes: #648784) + - When running commands in the chroot, redirect stdin from /dev/null. + - Stop using Popen.communicate() as it may run out of memory. + - Terminate commands producing more than 2 MB of output. (Closes: #649335) + - Create /etc/dpkg/dpkg.cfg.d/ if missing inside the chroot (e.g. on + lenny). (Closes: #647752) + - Remove logrotate and its dependencies only once. + - Only run 'apt-get update' after updating the sources.list. + - Only run 'apt-get clean' before creating tarballs or saving meta data. + - Do the same checks for running processes and broken symlinks in all + tests. (Closes: #648708) + - Create piupart's apt config in the chroot as /etc/apt.conf.d/piuparts + instead of /etc/apt.conf in order to allow overriding the settings from + custom scripts by just dropping new config bits in e.g. + /etc/apt/apt.conf.d/piuparts-foobar. + - Integrate diversion checks with Chroot.check_result(). + - Use 'apt-get remove' to remove all packages at once with proper + dependency ordering. (Closes: #648162) + - Purge all packages at once instead of doing it one-by-one. + - Make restore_selections() reinstall missing packages. (Closes: #648707) + - Set more environment variables to let custom scripts know where and + when they are being run: PIUPARTS_TEST, PIUPARTS_PHASE, + PIUPARTS_DISTRIBUTION{,_PREV,_NEXT}. See the README for details. + (Closes: #589498) + - Add missing post_install_* custom scripts to install_packages_by_name(). + (Closes: #628077) + - Remove pre_upgrade_* custom scripts, they can be replaced with + pre_install_* scripts that check for PIUPARTS_PHASE=upgrade. + - Add pre_test_* custom scripts. These are run exactly once at the + beginning of each test (after recording the initial chroot state). + - Allow multiple --scriptsdir options, the contents will be merged. + - Exclude /tmp/scripts when creating a tarball. + - Use --one-file-system when creating a tarball to exclude bindmounts etc. + - Restore base system from temp_tgz instead of running debootstrap again. + (Closes: #648193) + - Do not fail upgrade/distupgrade tests for a set of packages where not + all packages exist in the start distribution. This happens e.g. when + testing .changes files and packages were split/added. Only install the + old packages that actually exist according to 'apt-cache show'. + - Add --extra-old-packages option to intall additional old packages that + are not in the set of new packages to be tested. Useful for testing + upgrades with Conflicts/Replaces relationships, e.g. in package renames + or merges. + - Use consistent variable names for package lists. (Closes: #648177) + - Compute the changes in restore_selections(). + - Check for settings.scriptsdir inside run_scripts(). + - Consistently use chroot.relative() to build filenames inside the chroot. + * piupartslib/packagesdb.py: + - Handle 'unknown-preferred-alternative' state correctly. + - Add 'does-not-exist' state for dependency resolution to distinguish this + from 'unknown' state so that the latter only indicates 'unresolvable' or + 'not yet resolved'. + - Handle virtual packages separately from real packages. + - Use get_package_state() internally which 'resolves' (purely) virtual + packages by default (can be disabled). + * piuparts-master.py: + - Add a 'status' command that reports package counts for all states. + * piuparts-slave.py: + - Fix triggering tarball recreation. + - Check tarball age regularily. + - Log tarball creation in *.tgz.log. + - Request and print package counts from master. + - Reload section config every time a section is being run. + - Add precedence attribute to allow prioritizing different sections and to + suspend processing of low priority ones while there are packages waiting + in more important sections. + * piuparts-report.py: + - state-*.html: Sort package lists by name, display state of all + alternative dependencies and packages providing a virtual dependency. + - source/?/*.html: Sort binary packages by name. + - maintainer/?/*.html: Sort source packages by name. + - Update list of error states to be highlighted. + - Archive logs of packages that were removed from the distribution. + - Speedup generating maintainer summaries. + * Makefile: Use 'git describe' to get an exact stamp for development + versions. + + [ Dave Steele ] + * piuparts-slave.py: make Section.run() report the number of packages + processed and use this to decide whether a slave should sleep. + (Closes: #649967) + + [ Stefano Rivera ] + * piuparts.py: + - Properly install and remove logrotate. (Closes: #638832) + - Use eatmydata by default, add option --no-eatmydata. (This was discussed + in #633033.) + + -- Holger Levsen Fri, 23 Dec 2011 10:51:28 +0100 + piuparts (0.41ubuntu2) precise; urgency=low * Rebuild to drop python2.6 dependencies. @@ -47,7 +206,7 @@ environment variable, the latter overwriting the former (if present) - Thanks to Scott Schaefer for the patch. (Closes: #632046) - new option "--no-install-purge-test" to only do upgrade tests - - Thanks to Andreas Bergmann for the patch (Closes: #588482) + - Thanks to Andreas Beckmann for the patch (Closes: #588482) - run dpkg with --force-unsafe-io by default and introduce new option "--dpkg-noforce-unsafe-io" to disable this feature. (Closes: #633033) Thanks to Scott once more! @@ -923,3 +1082,4 @@ * First release of the Debian package. -- Lars Wirzenius Tue, 5 Jul 2005 20:08:00 +0300 + diff -Nru piuparts-0.41ubuntu2/debian/control piuparts-0.42ubuntu1/debian/control --- piuparts-0.41ubuntu2/debian/control 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/debian/control 2012-01-03 16:41:52.000000000 +0000 @@ -6,7 +6,7 @@ Priority: extra Standards-Version: 3.9.2 Build-Depends: debhelper (>=7) -Build-Depends-Indep: python (>=2.6.6-3~), asciidoc, xmlto +Build-Depends-Indep: python (>=2.6.6-3~), asciidoc, xmlto, git Homepage: http://piuparts.debian.org Vcs-Git: git://anonscm.debian.org/piuparts/piuparts.git Vcs-Browser: http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git @@ -15,7 +15,7 @@ Package: piuparts Architecture: all Depends: apt, ${python:Depends}, debootstrap, lsof, lsb-release, - python-debian, ${misc:Depends} + python-debian, ${misc:Depends}, python-debianbts, python-apt Suggests: python-rpy, ghostscript Description: .deb package installation, upgrading, and removal testing tool piuparts tests that .deb packages (as used by Debian) handle diff -Nru piuparts-0.41ubuntu2/debian/copyright piuparts-0.42ubuntu1/debian/copyright --- piuparts-0.41ubuntu2/debian/copyright 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/debian/copyright 2012-01-03 16:41:52.000000000 +0000 @@ -1,17 +1,35 @@ -This is piuparts, packaged for Debian by Lars Wirzenius . -Original author is also Lars Wirzenius. +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: piuparts +Upstream-Contact: Holger Levsen +Source: http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git +Copyright: 2005-2008 Lars Wirzenius + 2008-2011 Holger Levsen +Comment: Original author is also Lars Wirzenius + Upstream is bunch of lunatics who don't make release tarballs publicly + available. The Debian .orig.tar.gz is what would be released, though. -Upstream source repository: http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git -Upstream is bunch of lunatics who don't make release tarballs publicly -available. The Debian .orig.tar.gz is what would be released, though. +Files: * +Copyright: 2005-2008 Lars Wirzenius + 2008-2011 Holger Levsen +License: GPL-2+ + This program is free software; you can redistribute it + and/or modify it under the terms of the GNU General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later + version. + . + This program is distributed in the hope that it will be + useful, but WITHOUT ANY WARRANTY; without even the implied + warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the GNU General Public License for more + details. + . + You should have received a copy of the GNU General Public + License along with this package; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, + Boston, MA 02110-1301 USA + . + On Debian systems, the full text of the GNU General Public + License version 2 can be found in the file + '/usr/share/common-licenses/GPL-2'. -Copyright (C) 2005-2008 Lars Wirzenius and others - -piuparts is free software. You may copy it according to the -GNU General Public License, version 2, or at your option, any -later version. A copy of the license is not included, but you -can get one from most FTP sites that have GNU software, for -example, prep.ai.mit.edu. - -On a Debian GNU/Linux system, the GPL version 2 can be found in -/usr/share/common-licenses/GPL-2. diff -Nru piuparts-0.41ubuntu2/debian/.gitignore piuparts-0.42ubuntu1/debian/.gitignore --- piuparts-0.41ubuntu2/debian/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ piuparts-0.42ubuntu1/debian/.gitignore 2011-12-23 09:57:57.000000000 +0000 @@ -0,0 +1,5 @@ +files +piuparts.debhelper.log +piuparts.postinst.debhelper +piuparts.prerm.debhelper +piuparts.substvars diff -Nru piuparts-0.41ubuntu2/debian/NEWS piuparts-0.42ubuntu1/debian/NEWS --- piuparts-0.41ubuntu2/debian/NEWS 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/debian/NEWS 2012-01-03 16:41:52.000000000 +0000 @@ -1,12 +1,3 @@ -piuparts (0.41) unstable; urgency=low - - * Source code is now stored in git. - - http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git - git clone git+ssh://git.debian.org/git/piuparts/piuparts.git - - -- Holger Levsen Mon, 22 Aug 2011 15:04:43 +0200 - piuparts (0.38) unstable; urgency=low * New default behaviours in piuparts: diff -Nru piuparts-0.41ubuntu2/.gitignore piuparts-0.42ubuntu1/.gitignore --- piuparts-0.41ubuntu2/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ piuparts-0.42ubuntu1/.gitignore 2011-12-23 09:57:57.000000000 +0000 @@ -0,0 +1,6 @@ +*.pyc +piuparts +piuparts.1 +piuparts.1.html +README.html +docbook-xsl.css diff -Nru piuparts-0.41ubuntu2/Makefile piuparts-0.42ubuntu1/Makefile --- piuparts-0.41ubuntu2/Makefile 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/Makefile 2012-01-03 16:41:52.000000000 +0000 @@ -10,7 +10,7 @@ etcdir = $(prefix)/etc distribution=${shell dpkg-parsechangelog | sed -n 's/^Distribution: *//p'} ifeq ($(distribution),UNRELEASED) -version=${shell echo "`dpkg-parsechangelog | sed -n 's/^Version: *//p'`~`date +%Y%m%d%H%M`"} +version=${shell echo "`dpkg-parsechangelog | sed -n 's/^Version: *//p'`~`date +%Y%m%d%H%M`~`git describe --tags --dirty`"} else version=${shell dpkg-parsechangelog | sed -n 's/^Version: *//p'} endif @@ -42,11 +42,11 @@ echo $(version) sed -e 's/__PIUPARTS_VERSION__/$(version)/g' piuparts.py > piuparts install piuparts $(sbindir)/piuparts - + install -d $(sharedir)/piuparts for file in piuparts-slave piuparts-master piuparts-report piuparts-analyze; do \ install -m 0755 $$file.py $(sharedir)/piuparts/$$file ; done - + install -d $(site26)/piupartslib install -d $(site27)/piupartslib install -m 0644 piupartslib/*.py $(site26)/piupartslib diff -Nru piuparts-0.41ubuntu2/NEWS piuparts-0.42ubuntu1/NEWS --- piuparts-0.41ubuntu2/NEWS 2011-10-24 17:48:05.000000000 +0000 +++ piuparts-0.42ubuntu1/NEWS 2012-01-03 16:41:52.000000000 +0000 @@ -31,16 +31,16 @@ Piuparts now checks for symlinks whose target does not exist. - + Option parsing code has been rewritten, and --help now works better. - + The chroot is now minimized before used: all unnecessary packages are purged. - + /dev/MAKEDEV, /etc/nologin, /usr/doc/cpio, /var/spool/cron added to default ignores. - + A version number may now begin with a + character. There was a package that did that and piuparts crashed. @@ -52,11 +52,11 @@ The configuration files of piuparts-master/slave are now documented in the README. - + The Python profiler is no longer used. It used to be, but that was a leftover from development (also known as failure to read diffs before committing). - + When testing upgrades between distributions, piuparts now makes sure that the packages being tested are upgraded, even if it means removing an old version of a dependency. @@ -70,11 +70,11 @@ chroot after an installation or a purge has completed. This check then finds packages that don't use invoke-rc.d to start new processes. - + A number of new default ignores have been added: /etc/modprobe.d, compiled versions of debconf's Python modules, papercut, ssl certificates. - + /proc is now mounted (and unmounted) inside the chroot. @@ -94,15 +94,15 @@ for earlier versions of the same packages, and if so, moves them automatically around. This saves a bit of manual works. Thanks to Goswin Brederlow for the idea. - + When piuparts creates a chroot from a tarball (option -b), it now upgrades it before using it. A number of new entries to the default ignores list. - + Log files are now created with permissions that are 0666 modified with the process umask. - + piuparts-report.py has been optimized somewhat. Version 0.15 @@ -114,15 +114,15 @@ < and > (they're deprecated but one or two packges still use them). It also now allows underscores in package names because of the type-handling package. - + Small fixes to the manual page. - + New features and significant user visible changes piuparts-master now understands Provides: headers. - + A number of new entries to the default ignores list. - + New option --keep-sources-list from John Wright. @@ -133,12 +133,12 @@ Specifications for ignoring some directories were buggy and have now been fixed: /var/spool/news, /var/lib/cvs. - + When testing a .deb file given on the command line, if any of its dependencies were missing, the package itself would be removed instead of piuparts reporting an error. This has been fixed. - + The check for whether a package is marked untestable for piuparts-master now works. @@ -147,7 +147,7 @@ New program piuparts-report.py produces some "statistics" about packages and their status with regard to testing with piuparts-slave. - + The chroot is always set up for piuparts, even if it is unpacked from a tarball. This reduces problems with out-of-date chroots and with using the pbuilder base.tgz @@ -173,10 +173,10 @@ piuparts-report.py. Since these are not useful for most users, they're not installed on $PATH, but in /usr/share/piuparts instead. - + The slave part also runs upgrade tests between Debian releases, which run-piuparts.py didn't. - + Some additional files are ignored by default when comparing the state of the chroot before and after package installation. diff -Nru piuparts-0.41ubuntu2/piuparts.1.txt piuparts-0.42ubuntu1/piuparts.1.txt --- piuparts-0.41ubuntu2/piuparts.1.txt 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/piuparts.1.txt 2012-01-03 16:41:52.000000000 +0000 @@ -65,6 +65,12 @@ *--dpkg-noforce-unsafe-io*:: Prevent running dpkg with --force-unsafe-io. --force-unsafe-io causes dpkg to skip certain file system syncs known to cause substantial performance degradation on some filesystems. Thus, including this option reverts to safe but slower behavior. +*--no-eatmydata*:: + Prevent use of eatmydata. + +*--extra-old-packages*='pkg1[,pkg2]...':: + Install additional old packages before upgrading. Allows testing package renames/merges where the old package is no longer available in the new distribution and the new one utilizes Conflicts/Replaces. The argument is a comma separated list of package names and the option can be given multiple times. + *-i* 'filename', *--ignore*='filename':: Add a filename to the list of filenames to be ignored when comparing changes before and after installation. By default, piuparts ignores files that always change during a package installation and uninstallation, such as *dpkg* status files. The filename should be relative to the root of the chroot (e.g., _var/lib/dpkg/status_). This option can be used as many times as necessary. @@ -105,6 +111,9 @@ + Note that file: addresses works if the directories are made accessible from within the chroot with '--bindmount'. +*--no-diversions*:: + Don't check for broken diversions. + *-n, --no-ignores*:: Forget all built-in and other ignores that have been set so far. Any '-i' or '-I' arguments that come after this one will be obeyed, but none of the ones that come before. @@ -210,7 +219,7 @@ NOTES ----- -Outputs of commands run by piuparts are limited to the last megabyte. To change this limit, the source code needs to be edited. +Outputs of commands run by piuparts are limited to the last two megabytes. To change this limit, the source code needs to be edited. SEE ALSO -------- @@ -222,5 +231,5 @@ DATE ---- -2009-12-08 +2011-12-06 diff -Nru piuparts-0.41ubuntu2/piuparts-analyze.py piuparts-0.42ubuntu1/piuparts-analyze.py --- piuparts-0.41ubuntu2/piuparts-analyze.py 2011-10-24 17:48:05.000000000 +0000 +++ piuparts-0.42ubuntu1/piuparts-analyze.py 2012-01-03 16:41:52.000000000 +0000 @@ -1,6 +1,8 @@ #!/usr/bin/python +# -*- coding: utf-8 -*- # # Copyright 2005 Lars Wirzenius (liw@iki.fi) +# Copyright 2011 Mika Pflüger (debian@mikapflueger.de) # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -19,11 +21,12 @@ """Analyze failed piuparts logs and move them around when the errors are known. -This program looks at piuparts log files in ./fail, and compares them to -log files in ./bugged. If it finds one in ./fail that has the same -error messages as one in ./bugged, it copies the headers to the one in ./fail -and moves it to ./bugged. This is useful for repetitive uploads of the -same package that do not fix the problem. +This program looks at piuparts log files in ./fail, and queries the bts to find +out if bugs have been filed already. If so, it moves them to ./bugged. +It tries to detect if new versions of bugged packages are uploaded without solving +the bug and will then update the bts with the new found versions, and copy the +headers of the log in ./fail to the one in ./bugged and vice versa. It will then +move the failed log to ./bugged as well. """ @@ -31,19 +34,20 @@ import os import re import shutil -import sys +import subprocess +import debianbts +import apt_pkg -error_pattern = re.compile(r"(?<=\n)(\d+m\d+\.\d+s ERROR: .*\n( .*\n)*\n?)+") +apt_pkg.init_system() +error_pattern = re.compile(r"(?<=\n).*error.*\n?", flags=re.IGNORECASE) +chroot_pattern = re.compile(r"tmp/tmp.*?'") -def find_logs(dir): - return [os.path.join(dir, x) - for x in os.listdir(dir) if x.endswith(".log")] - -def package_name(log): - return os.path.basename(log).split("_", 1)[0] +def find_logs(directory): + return [os.path.join(directory, x) + for x in os.listdir(directory) if x.endswith(".log")] def find_bugged_logs(failed_log): @@ -52,40 +56,52 @@ return [x for x in find_logs("bugged") if pat in x] +def package_name(log): + return os.path.basename(log).split("_", 1)[0] + + +def package_version(log): + return os.path.basename(log).split("_", 1)[1].rstrip('.log') + + +def package_source_version(log): + version = package_version(log) + possible_binnmu_part = version.rsplit('+',1)[-1] + if possible_binnmu_part.startswith('b') and possible_binnmu_part[1:].isdigit(): + # the package version contains a binnmu-part which is not part of the source version + # and therefore not accepted/tracked by the bts. Remove it. + version = version.rsplit('+',1)[0] + return version + + def extract_errors(log): - f = file(log, "r") + """This pretty stupid implementation is basically just 'grep -i error', and then + removing the timestamps and the name of the chroot and the package version itself.""" + f = open(log) data = f.read() f.close() - m = error_pattern.search(data) - if m: - text = m.group() + whole = '' + pversion = package_version(log) + for match in error_pattern.finditer(data): + text = match.group() # Get rid of timestamps - text2 = [] - for line in text.split("\n"): - if line[:1].isdigit(): - text2.append(line.split(" ", 1)[1]) - else: - text2.append(line) - return "\n".join(text2) - else: - return None + if text[:1].isdigit(): + text = text.split(" ", 1)[1] + # Get rid of chroot names + if 'tmp/tmp' in text: + text = re.sub(chroot_pattern, "chroot'", text) + # Get rid of the package version + text = text.replace(pversion, '') + whole += text + return whole def extract_headers(log): - f = file(log, "r") + f = open(log) data = f.read() f.close() headers = [] - cont = False - for line in data.split("\n\n", 1)[0].split("\n"): - if line.startswith("Start:"): - cont = True - elif cont and line[:1].isspace(): - pass - else: - headers.append(line) - cont = False - headers = "\n".join(headers) + headers = data.partition("\nExecuting:")[0] if headers and not headers.endswith("\n"): headers += "\n" return headers @@ -95,40 +111,80 @@ f = file(filename, "r") old_data = f.read() f.close() - f = file(filename + ".tmp", "w") f.write(data) f.write(old_data) f.close() - + shutil.copymode(filename, filename + ".tmp") - + os.rename(filename, filename + "~") os.rename(filename + ".tmp", filename) os.remove(filename + "~") -def mark_bugged(failed_log, bugged_log): - print "Moving", failed_log, "to bugged" +def get_bug_versions(bug): + """Gets a list of only the version numbers for which the bug is found. + Newest versions are returned first.""" + # debianbts returns it in the format package/1.2.3 or 1.2.3 which will become 1.2.3 + return reversed(sorted([x.rsplit('/', 1)[-1] for x in debianbts.get_status((bug,))[0].found_versions], cmp=apt_pkg.version_compare)) + + +def move_to_bugged(failed_log): + print("Moving %s to bugged" % failed_log) + os.rename(failed_log, os.path.join("bugged", os.path.basename(failed_log))) + + +def mark_bugged_version(failed_log, bugged_log): + """Copies the headers from the old log to the new log and vice versa and + moves the new log to bugged. Removes the old log in bugged.""" + bugged_headers = extract_headers(bugged_log) + failed_headers = extract_headers(failed_log) + prepend_to_file(failed_log, bugged_headers) + prepend_to_file(bugged_log, failed_headers) + move_to_bugged(failed_log) - headers = extract_headers(bugged_log) - prepend_to_file(failed_log, headers) - if os.path.isdir(".bzr"): - assert "'" not in failed_log - os.system("bzr move '%s' bugged" % failed_log) - else: - os.rename(failed_log, - os.path.join("bugged", os.path.basename(failed_log))) +def bts_update_found(bugnr, newversion): + # Disabled for now as I'm not sure automatic bug updating is wanted + #subprocess.check_call(('bts', 'found', bugnr, newversion)) + print(' '.join(('bts', 'found', str(bugnr), newversion))) -def mark_logs_with_known_bugs(): + +def mark_logs_with_reported_bugs(): for failed_log in find_logs("fail"): + pname = package_name(failed_log) + pversion = package_source_version(failed_log) failed_errors = extract_errors(failed_log) - for bugged_log in find_bugged_logs(failed_log): - bugged_errors = extract_errors(bugged_log) - if failed_errors == bugged_errors: - mark_bugged(failed_log, bugged_log) - break + moved = False + for bug in piuparts_bugs_in(pname): + for bug_version in get_bug_versions(bug): + + if apt_pkg.version_compare(pversion, bug_version) == 0: # pversion == bug_version + if not moved: + move_to_bugged(failed_log) + moved = True + break + + elif apt_pkg.version_compare(pversion, bug_version) > 0: # pversion > bug_version + bugged_logs = find_bugged_logs(failed_log) + if not bugged_logs and not moved: + print('%s/%s: Maybe the bug was filed earlier: %d against %s/%s' + % (pname, pversion, bug, pname, bug_version)) + break + for bugged_log in bugged_logs: + old_pversion = package_source_version(bugged_log) + bugged_errors = extract_errors(bugged_log) + if (apt_pkg.version_compare(old_pversion, bug_version) == 0 # old_pversion == bug_version + and + failed_errors == bugged_errors): + # a bug was filed for an old version of the package, + # and the errors were the same back then - assume it is the same bug. + if not moved: + mark_bugged_version(failed_log, bugged_log) + moved = True + bts_update_found(bug, pversion) + break def report_packages_with_many_logs(): @@ -154,10 +210,24 @@ print +piuparts_usertags_cache = None +def all_piuparts_bugs(): + global piuparts_usertags_cache + if piuparts_usertags_cache is None: + piuparts_usertags_cache = debianbts.get_usertag("debian-qa@lists.debian.org", 'piuparts')['piuparts'] + return piuparts_usertags_cache + + +def piuparts_bugs_in(package): + return debianbts.get_bugs('package', package, 'bugs', all_piuparts_bugs()) + + def main(): - mark_logs_with_known_bugs() + mark_logs_with_reported_bugs() report_packages_with_many_logs() if __name__ == "__main__": main() + +# vi:set et ts=4 sw=4 : diff -Nru piuparts-0.41ubuntu2/piupartslib/conf.py piuparts-0.42ubuntu1/piupartslib/conf.py --- piuparts-0.41ubuntu2/piupartslib/conf.py 2011-10-24 17:48:05.000000000 +0000 +++ piuparts-0.42ubuntu1/piupartslib/conf.py 2012-01-03 16:41:52.000000000 +0000 @@ -40,7 +40,7 @@ for key, value in defaults.iteritems(): self[key] = value self._mandatory = mandatory - + def read(self, filename): cp = ConfigParser.ConfigParser() cp.read(filename) @@ -49,3 +49,5 @@ self[key] = cp.get(self._section, key) elif key in self._mandatory: raise MissingMandatorySetting(filename, key) + +# vi:set et ts=4 sw=4 : diff -Nru piuparts-0.41ubuntu2/piupartslib/dependencyparser.py piuparts-0.42ubuntu1/piupartslib/dependencyparser.py --- piuparts-0.41ubuntu2/piupartslib/dependencyparser.py 2011-10-24 17:48:05.000000000 +0000 +++ piuparts-0.42ubuntu1/piupartslib/dependencyparser.py 2012-01-03 16:41:52.000000000 +0000 @@ -37,10 +37,10 @@ self._msg = "Error: %s: %s (text at error: '%s', full text being parsed: '%s')" % \ (cursor.get_position(), msg, cursor.get_text(10), cursor.get_full_text()) - + def __str__(self): return self._msg - + def __repr__(self): return self._msg @@ -48,7 +48,7 @@ class _Cursor: """Store an input string and a movable location in it""" - + def __init__(self, input): self._input = input self._len = len(self._input) @@ -62,7 +62,7 @@ """Are we at the end of the input?""" self.skip_whitespace() return self._pos >= self._len - + def next(self): """Move to the next character""" if self._pos < self._len: @@ -87,9 +87,9 @@ def match(self, regexp): """Match a regular expression against the current position - + The cursor is advanced by the length of the match, if any. - + """ m = regexp.match(self._input[self._pos:]) if m: @@ -98,10 +98,10 @@ def match_literal(self, literal): """Match a literal string against the current position. - + Return True and move position if there is a match, else return False. - + """ if self.get_text(len(literal)) == literal: self._pos += len(literal) @@ -132,17 +132,17 @@ class DependencyParser: """Parse Debian package relationship strings - + Debian packages have a rich language for expressing their relationships. See the Debian Policy Manual, chapter 7 ("Declaring relationships between packages"). This Python module implements a parser for strings expressing such relationships. - + Syntax of dependency fields (Pre-Depends, Depends, Recommends, Suggests, Conflicts, Provides, Replaces, Enhances, Build-Depends, Build-Depends-Indep, Build-Conflicts, Build-Conflicts-Indep), in a BNF-like form: - + depends-field ::= EMPTY | dependency ("," dependency)* dependency ::= possible-dependency ("|" possible-dependency)* possible-dependency ::= package-name version-dependency? @@ -168,18 +168,18 @@ name-char ::= alphanumeric | "+" | "-" | "." | "_" version-char ::= alphanumeric | "." | "+" | "-" | ":" | "~" debian-version-char ::= alphanumeric | "." | "+" - + White space can occur between any tokens except inside package-name, version-number, or arch-name. Some of the headers restrict the syntax somewhat, e.g., Provides does not allow version-dependency, but this is not included in the syntax for simplicity. - + Note: Added "_" to name-char, because some packages (type-handling in particular) use Provides: headers with bogus package names. - + Note: Added upper case letters to name pattern, since it some of the Mozilla localization packages use or used them. - + """ def __init__(self, input_string): @@ -188,19 +188,19 @@ def get_dependencies(self): """Return parsed dependencies - + The result is a list of lists of SimpleDependency objects. Let's try that again. - + The result is a list of dependencies, corresponding to the comma-separated items in the dependency list. Each dependency is also a list, or SimpleDependency objects, representing alternative ways to fulfill the dependency; in other words, items separated by the vertical bar (|). - + For example, "foo, bar | foobar" would result in the following list: [[foo], [bar, foobar]]. - + """ return self._list @@ -227,7 +227,7 @@ break dep = self._parse_possible_dependency() return vlist - + def _parse_possible_dependency(self): name = self._parse_package_name() if not name: @@ -290,7 +290,7 @@ self._cursor.skip_whitespace() if self._cursor.get_char() == "[": self._cursor.next() - + vlist = [] while True: self._cursor.skip_whitespace() @@ -302,7 +302,9 @@ raise DependencySyntaxError("Expected architecture name", self._cursor) vlist.append(m.group()) - + return vlist else: return None + +# vi:set et ts=4 sw=4 : diff -Nru piuparts-0.41ubuntu2/piupartslib/__init__.py piuparts-0.42ubuntu1/piupartslib/__init__.py --- piuparts-0.41ubuntu2/piupartslib/__init__.py 2011-10-24 17:48:05.000000000 +0000 +++ piuparts-0.42ubuntu1/piupartslib/__init__.py 2012-01-03 16:41:52.000000000 +0000 @@ -39,3 +39,5 @@ socket.close() bzfile.seek(0) return bzfile + +# vi:set et ts=4 sw=4 : diff -Nru piuparts-0.41ubuntu2/piupartslib/packagesdb.py piuparts-0.42ubuntu1/piupartslib/packagesdb.py --- piuparts-0.41ubuntu2/piupartslib/packagesdb.py 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/piupartslib/packagesdb.py 2012-01-03 16:41:52.000000000 +0000 @@ -26,7 +26,6 @@ import dircache import os -import random import tempfile import UserDict @@ -45,44 +44,6 @@ else: headers.append(line) return headers - -def unique (s): - # taken from http://code.activestate.com/recipes/52560/ - thanks to Tim Peters - n = len(s) - if n == 0: - return [] - - u = {} - try: - for x in s: - u[x] = 1 - except TypeError: - del u # move on to the next method - else: - return u.keys() - - try: - t = list(s) - t.sort() - except TypeError: - del t # move on to the next method - else: - assert n > 0 - last = t[0] - lasti = i = 1 - while i < n: - if t[i] != last: - t[lasti] = last = t[i] - lasti += 1 - i += 1 - return t[:lasti] - - # Brute force is all that's left. - u = [] - for x in s: - if x not in u: - u.append(x) - return u class Package(UserDict.UserDict): @@ -93,7 +54,8 @@ name, value = header.split(":", 1) self[name.strip()] = value.strip() self._parsed_deps = {} - + self._parsed_alt_deps = {} + def _parse_dependencies(self, header_name): if header_name in self._parsed_deps: depends = self._parsed_deps[header_name] @@ -104,6 +66,17 @@ self._parsed_deps[header_name] = depends return depends + def _parse_alternative_dependencies(self, header_name): + if header_name in self._parsed_alt_deps: + depends = self._parsed_alt_deps[header_name] + else: + parser = DependencyParser(self[header_name]) + depends = parser.get_dependencies() + depends = [[alt.name for alt in alternatives] for alternatives in depends] + self._parsed_alt_deps[header_name] = depends + return depends + + # first alternative only - [package_name...] def dependencies(self): vlist = [] for header in ["Depends", "Pre-Depends"]: @@ -111,6 +84,14 @@ vlist += self._parse_dependencies(header) return vlist + # all alternatives - [[package_name...]...] + def all_dependencies(self): + vlist = [] + for header in ["Depends", "Pre-Depends"]: + if header in self: + vlist += self._parse_alternative_dependencies(header) + return vlist + def depends_with_alts(self, header_name): vlist = [] if header_name in self: @@ -160,7 +141,7 @@ def listdir(self, dirname): return dircache.listdir(dirname) - + def exists(self, pathname): try: cache = self.exists_cache @@ -170,10 +151,10 @@ if pathname not in cache: cache[pathname] = os.path.exists(pathname) return cache[pathname] - + def open_file(self, pathname, mode): return file(pathname, mode) - + def remove_file(self, pathname): os.remove(pathname) @@ -186,7 +167,7 @@ if self.exists(os.path.join(subdir, log_name)): return True return False - + def any_log_exists(self, package, subdirs): try: cache = self.basename_cache @@ -202,11 +183,11 @@ if len(parts) == 2 and parts[0] == package_name: return True return False - + def create(self, subdir, package, version, contents): (fd, temp_name) = tempfile.mkstemp(dir=subdir) os.close(fd) - + # tempfile.mkstemp sets the file mode to be readable only by owner. # Let's make it follow the umask. umask = os.umask(0) @@ -229,10 +210,11 @@ full_name = os.path.join(subdir, self._log_name(package, version)) if self.exists(full_name): self.remove_file(full_name) - + class PackagesDB: + # keep in sync with piuparts-report.py: emphasize_reason() _states = [ "successfully-tested", "failed-testing", @@ -244,11 +226,12 @@ "dependency-cannot-be-tested", "dependency-does-not-exist", "circular-dependency", + #"does-not-exist", # can only happen as query result for a dependency "unknown", "unknown-preferred-alternative", "no-dependency-from-alternatives-exists", ] - + _dep_state_to_state = { "failed-testing": "dependency-failed-testing", "cannot-be-tested": "dependency-cannot-be-tested", @@ -258,6 +241,7 @@ "dependency-cannot-be-tested": "dependency-cannot-be-tested", "dependency-does-not-exist": "dependency-does-not-exist", "circular-dependency": "circular-dependency", + "does-not-exist": "dependency-does-not-exist", "unknown-preferred-alternative": "unknown-preferred-alternative", "no-dependency-from-alternatives-exists": "dependency-cannot-be-tested", } @@ -272,7 +256,7 @@ self._package_state = {} self.set_subdirs(ok="pass", fail="fail", evil="untestable", reserved="reserved", morefail=["bugged"]) - + def set_subdirs(self, ok=None, fail=None, evil=None, reserved=None, morefail=None): # Prefix all the subdirs with the prefix if self.prefix: @@ -290,12 +274,12 @@ if morefail: self._morefail = [pformat % s for s in morefail] self._all = [self._ok, self._fail, self._evil, self._reserved] + self._morefail - + def create_subdirs(self): for sdir in self._all: if not os.path.exists(sdir): os.makedirs(sdir) - + def read_packages_file(self, input): self._packages_files.append(PackagesFile(input)) self._packages = None @@ -307,13 +291,16 @@ def _find_all_packages(self): if self._packages is None: self._packages = {} + self._virtual_packages = {} for pf in self._packages_files: for p in pf.values(): self._packages[p["Package"]] = p for p in self._packages.values(): for provided in p.provides(): - if provided not in self._packages: - self._packages[provided] = p + if provided != p["Package"]: + if provided not in self._virtual_packages: + self._virtual_packages[provided] = [] + self._virtual_packages[provided].append(p["Package"]) def _get_recursive_dependencies(self, package, break_circles=True): assert self._packages is not None @@ -326,13 +313,15 @@ deps.append(dep) if dep in self._packages: more += self._packages[dep].dependencies() - + elif dep in self._virtual_packages: + more += self._packages[self._virtual_packages[dep][0]].dependencies() + # Break circular dependencies if break_circles and package["Package"] in deps: deps.remove(package["Package"]) return deps - + def _compute_package_state(self, package): if self._logdb.log_exists(package, [self._ok]): return "successfully-tested" @@ -370,9 +359,9 @@ prefer_alt = None for alternative in alt_deps[d]: dep = alternative.name - if dep in self._package_state: + altdep_state = self.get_package_state(dep) + if altdep_state != "does-not-exist": alt_found += 1 - altdep_state = self._package_state[dep] if prefer_alt_score < 3 and altdep_state == "essential-required": prefer_alt = alternative prefer_alt_idx = d @@ -409,18 +398,15 @@ return state for dep in package.dependencies(): - if dep not in self._package_state: - return "dependency-does-not-exist" - dep_state = self._package_state[dep] - if dep_state is None: - return "unknown" - elif dep_state in self._dep_state_to_state: + dep_state = self.get_package_state(dep) + if dep_state in self._dep_state_to_state: return self._dep_state_to_state[dep_state] state = "waiting-to-be-tested" for dep in package.dependencies(): - if self._package_state[dep] not in \ - ["successfully-tested", "essential-required"]: + dep_state = self.get_package_state(dep) + if dep_state not in \ + ["successfully-tested", "essential-required"]: state = "unknown" break if state == "waiting-to-be-tested": @@ -432,26 +418,27 @@ if pkg in deps: deps.remove(pkg) if package["Package"] in deps: - return "circular-dependency" # actually, it's a unknown circular-dependency - + return "circular-dependency" # actually, it's an unknown circular-dependency + # treat circular-dependencies as testable (for the part of the circle) state = "unknown" if package["Package"] in self._known_circular_depends: - for dep in package.dependencies(): - if dep not in self._known_circular_depends and self._package_state[dep] not in \ - ["successfully-tested", "essential-required"]: - state = "unknown" - break - if dep in self._known_circular_depends and self._package_state[dep] not in \ - ["failed-testing","dependency-failed-testing"]: - state = "waiting-to-be-tested" - continue + for dep in package.dependencies(): + dep_state = self.get_package_state(dep) + if dep not in self._known_circular_depends and dep_state not in \ + ["successfully-tested", "essential-required"]: + state = "unknown" + break + if dep in self._known_circular_depends and dep_state not in \ + ["failed-testing", "dependency-failed-testing"]: + state = "waiting-to-be-tested" + continue return state def _compute_package_states(self): if self._in_state is not None: return - + todo = [] unpreferred_alt = [] @@ -493,7 +480,9 @@ self._in_state["unknown"] = todo self._in_state["unknown-preferred-alternative"] = unpreferred_alt - + for package_name in unpreferred_alt: + self._package_state[package_name] = "unknown-preferred-alternative" + for state in self._states: self._in_state[state].sort() @@ -502,12 +491,20 @@ def get_pkg_names_in_state(self, state): self._compute_package_states() - return self._in_state[state] - - def get_packages_in_state(self, state): - self._compute_package_states() - return unique([self._packages[name] for name in self._in_state[state]]) - + return set(self._in_state[state]) + + def has_package(self, name): + self._find_all_packages() + return name in self._packages + + def get_package(self, name): + return self._packages[name] + + def get_providers(self, name): + if name in self._virtual_packages: + return self._virtual_packages[name] + return [] + def get_all_packages(self): self._find_all_packages() return self._packages @@ -538,24 +535,25 @@ else: return self._packages[package_name][header] - def get_package_state(self, package_name): - return self._package_state[package_name] - - def state_by_name(self, package_name): + def get_package_state(self, package_name, resolve_virtual=True): if package_name in self._package_state: return self._package_state[package_name] - else: - return "unknown" + if package_name in self._virtual_packages: + if resolve_virtual: + provider = self._virtual_packages[package_name][0] + return self._package_state[provider] + else: + return "virtual" + return "does-not-exist" def _find_packages_ready_for_testing(self): - return self.get_packages_in_state("waiting-to-be-tested") + return self.get_pkg_names_in_state("waiting-to-be-tested") def reserve_package(self): - plist = self._find_packages_ready_for_testing() - random.shuffle(plist) - for p in plist: - if self._logdb.create(self._reserved, p["Package"], - p["Version"], ""): + pset = self._find_packages_ready_for_testing() + while (len(pset)): + p = self.get_package(pset.pop()) + if self._logdb.create(self._reserved, p["Package"], p["Version"], ""): return p return None @@ -594,3 +592,5 @@ else: raise Exception("Log file exists already: %s (%s)" % (package, version)) + +# vi:set et ts=4 sw=4 : diff -Nru piuparts-0.41ubuntu2/piuparts-master.py piuparts-0.42ubuntu1/piuparts-master.py --- piuparts-0.41ubuntu2/piuparts-master.py 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/piuparts-master.py 2012-01-03 16:41:52.000000000 +0000 @@ -79,7 +79,7 @@ line = self._input.readline() logging.debug(">> " + line.rstrip()) return line - + def _writeline(self, line): logging.debug("<< " + line) self._output.write(line + "\n") @@ -114,6 +114,7 @@ def __init__(self, input, output, packages_file, known_circular_depends="", section=None): Protocol.__init__(self, input, output) self._commands = { + "status": self._status, "reserve": self._reserve, "unreserve": self._unreserve, "pass": self._pass, @@ -152,8 +153,19 @@ (count, command, " ".join(args))) def dump_pkgs(self): for st in self._binary_db.get_states(): - for pkg in self._binary_db.get_pkg_names_in_state(st): - logging.debug("%s : %s\n" % (st,pkg)) + for name in self._binary_db.get_pkg_names_in_state(st): + logging.debug("%s : %s\n" % (st,name)) + + def _status(self, command, args): + self._check_args(0, command, args) + stats = "" + total = 0 + for state in self._binary_db.get_states(): + count = len(self._binary_db.get_pkg_names_in_state(state)) + total += count + stats += "%s=%d " % (state, count) + stats += "total=%d" % total + self._short_response("ok", stats) def _reserve(self, command, args): self._check_args(0, command, args) @@ -198,12 +210,12 @@ section = sys.argv[1] config = Config(section=section) config.read(CONFIG_FILE) - + setup_logging(logging.DEBUG, config["log-file"]) if not os.path.exists(os.path.join(master_directory, section)): os.makedirs(os.path.join(master_directory, section)) - + logging.info("Fetching %s" % config["packages-url"]) packages_file = piupartslib.open_packages_url(config["packages-url"]) known_circular_depends = config["known_circular_depends"] @@ -218,3 +230,5 @@ if __name__ == "__main__": main() + +# vi:set et ts=4 sw=4 : diff -Nru piuparts-0.41ubuntu2/piuparts.py piuparts-0.42ubuntu1/piuparts.py --- piuparts-0.41ubuntu2/piuparts.py 2011-10-24 17:53:50.000000000 +0000 +++ piuparts-0.42ubuntu1/piuparts.py 2012-01-04 10:08:57.000000000 +0000 @@ -49,6 +49,7 @@ import unittest import urllib import uuid +from signal import signal, SIGTERM, SIGKILL try: from debian import deb822 @@ -58,13 +59,13 @@ class Defaults: """Default settings which depend on flavor of Debian. - + Some settings, such as the default mirror and distribution, depend on which flavor of Debian we run under: Debian itself, or a derived distribution such as Ubuntu. This class abstracts away the defaults so that the rest of the code can just refer to the values defined herein. - + """ def get_components(self): @@ -72,10 +73,10 @@ def get_mirror(self): """Return default mirror.""" - + def get_distribution(self): """Return default distribution.""" - + class DebianDefaults(Defaults): @@ -104,13 +105,13 @@ class DefaultsFactory: """Instantiate the right defaults class.""" - + def guess_flavor(self): p = subprocess.Popen(["lsb_release", "-i", "-s"], stdout=subprocess.PIPE) stdout, stderr = p.communicate() return stdout.strip().lower() - + def new_defaults(self): if not settings.defaults: settings.defaults = self.guess_flavor() @@ -126,11 +127,11 @@ class Settings: """Global settings for this program.""" - + def __init__(self): self.defaults = None self.tmpdir = None - self.scriptsdir = None + self.scriptsdirs = [] self.keep_tmpdir = False self.single_changes_list = False # limit output of logfiles to the last megabyte: @@ -151,8 +152,10 @@ self.list_installed_files = False self.no_install_purge_test = False self.no_upgrade_test = False + self.extra_old_packages = [] self.skip_cronfiles_test = False self.skip_logrotatefiles_test = False + self.check_broken_diversions = True self.check_broken_symlinks = True self.warn_broken_symlinks = True self.debfoster_options = None @@ -165,6 +168,7 @@ "/etc/apt/trustdb.gpg", "/etc/apt/trusted.gpg", "/etc/apt/trusted.gpg~", + "/etc/blkid.tab", "/etc/crypttab", "/etc/exports", "/etc/group", @@ -252,6 +256,7 @@ "/usr/lib/python2\../site-packages/debconf.py[co]", "/var/backups/.*", "/var/cache/man/(/.*)?", + "/var/lib/apt/lists/.*", "/var/lib/cvs(/.*)?", "/var/lib/dpkg/alternatives", "/var/lib/dpkg/triggers/.*", @@ -323,7 +328,7 @@ handler.setFormatter(formatter) logger.addHandler(handler) HANDLERS.append(handler) - + if log_file_name: handler = logging.FileHandler(log_file_name) handler.setFormatter(formatter) @@ -379,9 +384,30 @@ env["LC_ALL"] = "C" env["LANGUAGES"] = "" env["PIUPARTS_OBJECTS"] = ' '.join(str(vobject) for vobject in settings.testobjects ) - p = subprocess.Popen(command, env=env, stdin=subprocess.PIPE, + devnull = open('/dev/null', 'r') + p = subprocess.Popen(command, env=env, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - (output, _) = p.communicate() + output = "" + excessive_output = False + while p.poll() is None: + """Read 64 KB chunks, but depending on the output buffering behavior + of the command we may get less even if more output is coming later. + Abort after reading 2 MB.""" + output += p.stdout.read(1 << 16) + if (len(output) > (1 << 21)): + excessive_output = True + logging.error("Terminating command due to excessive output") + p.terminate() + for i in range(10): + time.sleep(0.5) + if p.poll() is not None: + break + else: + logging.error("Killing command due to excessive output") + p.kill() + p.wait() + break + devnull.close() if output: dump("\n" + indent_string(output.rstrip("\n"))) @@ -394,6 +420,8 @@ else: logging.error("Command failed (status=%d): %s\n%s" % (p.returncode, repr(command), indent_string(output))) + if excessive_output: + logging.error("Command was terminated while producing excessive output") panic() return p.returncode, output @@ -440,7 +468,7 @@ def make_metapackage(name, depends, conflicts): """Return the path to a .deb created just for satisfying dependencies - + Caller is responsible for removing the temporary directory containing the .deb when finished. """ @@ -472,32 +500,78 @@ return os.path.join(tmpdir, name) + '.deb' -def is_broken_symlink(root, dirpath, filename): - """Is symlink dirpath+filename broken? - - When resolving the symlink, pretend (similar to chroot) that root is - the root of the filesystem. Note that this does NOT work completely - correctly if the symlink target contains .. path components. This is - good enough for my immediate purposes, but nowhere near good enough - for anything that needs to be secure. For that, use chroot and have - the kernel resolve symlinks instead. +def split_path(pathname): + parts = [] + while pathname: + (head, tail) = os.path.split(pathname) + #print "split '%s' => '%s' + '%s'" % (pathname, head, tail) + if tail: + parts.append(tail) + elif not head: + break + elif head == pathname: + parts.append(head) + break + pathname = head + return parts + +def canonicalize_path(root, pathname): + """Canonicalize a path name, simulating chroot at 'root'. + + When resolving the symlink, pretend (similar to chroot) that + 'root' is the root of the filesystem. Also resolve '..' and + '.' components. This should not escape the chroot below + 'root', but for security concerns, use chroot and have the + kernel resolve symlinks instead. """ - - pathname = os.path.join(dirpath, filename) - i = 0 - while os.path.islink(pathname): - if i >= 10: # let's avoid infinite loops... - return True - i += 1 - target = os.readlink(pathname) - if os.path.isabs(target): - pathname = os.path.join(root, target[1:]) # Assume Unix filenames + #print "\nCANONICALIZE %s %s" % (root, pathname) + seen = [] + parts = split_path(pathname) + #print "PARTS ", list(reversed(parts)) + path = "/" + while parts: + tag = "\n".join(parts + [path]) + #print "TEST '%s' + " % path, list(reversed(parts)) + if tag in seen or len(seen) > 1024: + fullpath = os.path.join(path, *reversed(parts)) + #print "LOOP %s" % fullpath + path = fullpath + logging.error("ELOOP: Too many symbolic links in '%s'" % path) + break + seen.append(tag) + part = parts.pop() + # Using normpath() to cleanup '.', '..' and multiple slashes. + # Removing a suffix 'foo/..' is safe here since it can't change the + # meaning of 'path' because it contains no symlinks - they have been + # resolved already. + newpath = os.path.normpath(os.path.join(path, part)) + rootedpath = os.path.join(root, newpath[1:]) + if newpath == "/": + path = "/" + elif os.path.islink(rootedpath): + target = os.readlink(rootedpath) + #print "LINK to '%s'" % target + if os.path.isabs(target): + path = "/" + parts.extend(split_path(target)) else: - pathname = os.path.join(os.path.dirname(pathname), target) + path = newpath + #print "FINAL '%s'" % path + return path + + +def is_broken_symlink(root, dirpath, filename): + """Is symlink dirpath+filename broken?""" + + if dirpath[:len(root)] == root: + dirpath = dirpath[len(root):] + pathname = canonicalize_path(root, os.path.join(dirpath, filename)) + pathname = os.path.join(root, pathname[1:]) # The symlink chain, if any, has now been resolved. Does the target # exist? + #print "EXISTS ", pathname, os.path.exists(pathname) return not os.path.exists(pathname) @@ -521,10 +595,25 @@ self.symlink("absolute-broken", "absolute-broken-to-symlink") self.symlink("/", "absolute-works") self.symlink("/absolute-works", "absolute-works-to-symlink") - + os.mkdir(os.path.join(self.testdir, "dir")) + self.symlink("dir", "dir-link") + os.mkdir(os.path.join(self.testdir, "dir/subdir")) + self.symlink("subdir", "dir/subdir-link") + self.symlink("notexist/", "trailing-slash-broken") + self.symlink("dir/", "trailing-slash-works") + self.symlink("selfloop", "selfloop") + self.symlink("/absolute-selfloop", "absolute-selfloop") + self.symlink("../dir/selfloop", "dir/selfloop") + self.symlink("../dir-link/selfloop", "dir/selfloop1") + self.symlink("../../dir/subdir/selfloop", "dir/subdir/selfloop") + self.symlink("../../dir-link/subdir/selfloop", "dir/subdir/selfloop1") + self.symlink("../../link/subdir-link/selfloop", "dir/subdir/selfloop2") + self.symlink("../../dir-link/subdir-link/selfloop", "dir/subdir/selfloop3") + self.symlink("explode/bomb", "explode") + def tearDown(self): shutil.rmtree(self.testdir) - + def testRelativeBroken(self): self.failUnless(is_broken_symlink(self.testdir, self.testdir, "relative-broken")) @@ -540,7 +629,37 @@ def testAbsoluteBrokenToSymlink(self): self.failUnless(is_broken_symlink(self.testdir, self.testdir, "absolute-broken-to-symlink")) - + + def testTrailingSlashBroken(self): + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "trailing-slash-broken")) + + def testSelfLoopBroken(self): + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "selfloop")) + + def testExpandingSelfLoopBroken(self): + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "explode")) + + def testAbsoluteSelfLoopBroken(self): + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "absolute-selfloop")) + + def testSubdirSelfLoopBroken(self): + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "dir/selfloop")) + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "dir/selfloop1")) + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "dir/subdir/selfloop")) + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "dir/subdir/selfloop1")) + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "dir/subdir/selfloop2")) + self.failUnless(is_broken_symlink(self.testdir, self.testdir, + "dir/subdir/selfloop3")) + def testRelativeWorks(self): self.failIf(is_broken_symlink(self.testdir, self.testdir, "relative-works")) @@ -557,6 +676,10 @@ self.failIf(is_broken_symlink(self.testdir, self.testdir, "absolute-works-to-symlink")) + def testTrailingSlashWorks(self): + self.failIf(is_broken_symlink(self.testdir, self.testdir, + "trailing-slash-works")) + def testMultiLevelNestedSymlinks(self): # target/first-link -> ../target/second-link -> ../target @@ -566,55 +689,70 @@ self.failIf(is_broken_symlink(self.testdir, self.testdir, "target/first-link")) + def testMultiLevelNestedAbsoluteSymlinks(self): + # first-link -> /second-link/final-target + # second-link -> /target-dir + + os.mkdir(os.path.join(self.testdir, "final-dir")) + os.mkdir(os.path.join(self.testdir, "final-dir/final-target")) + self.symlink("/second-link/final-target", "first-link") + self.symlink("/final-dir", "second-link") + self.failIf(is_broken_symlink(self.testdir, self.testdir, + "first-link")) + class Chroot: """A chroot for testing things in.""" - + def __init__(self): self.name = None - + + self.pre_install_diversions = None + def create_temp_dir(self): """Create a temporary directory for the chroot.""" self.name = tempfile.mkdtemp(dir=settings.tmpdir) os.chmod(self.name, 0755) logging.debug("Created temporary directory %s" % self.name) - def create(self): + def create(self, temp_tgz = None): """Create a chroot according to user's wishes.""" self.create_temp_dir() cid = do_on_panic(self.remove) - if settings.basetgz: + if temp_tgz: + self.unpack_from_tgz(temp_tgz) + elif settings.basetgz: self.unpack_from_tgz(settings.basetgz) elif settings.lvm_volume: self.setup_from_lvm(settings.lvm_volume) else: self.setup_minimal_chroot() - self.configure_chroot() self.mount_proc() self.mount_selinux() + self.configure_chroot() if settings.basetgz: self.run(["apt-get", "-yf", "upgrade"]) self.minimize() - self.run(["apt-get", "clean"]) - #copy scripts dir into the chroot - if settings.scriptsdir is not None: + # Copy scripts dirs into the chroot, merging all dirs together, + # later files overwriting earlier ones. + if settings.scriptsdirs: dest = self.relative("tmp/scripts/") if not os.path.exists(self.relative("tmp/scripts/")): os.mkdir(dest) - logging.debug("Copying scriptsdir to %s" % dest) - for sfile in os.listdir(settings.scriptsdir): - if (sfile.startswith("post_") or sfile.startswith("pre_")) and os.path.isfile(os.path.join((settings.scriptsdir), sfile)): - shutil.copy(os.path.join((settings.scriptsdir), sfile), dest) + for sdir in settings.scriptsdirs: + logging.debug("Copying scriptsdir %s to %s" % (sdir, dest)) + for sfile in os.listdir(sdir): + if (sfile.startswith("post_") or sfile.startswith("pre_")) and os.path.isfile(os.path.join(sdir, sfile)): + shutil.copy(os.path.join(sdir, sfile), dest) # Run custom scripts after creating the chroot. - if settings.scriptsdir is not None: - self.run_scripts("post_setup") + self.run_scripts("post_setup") - if settings.savetgz: + if settings.savetgz and not temp_tgz: self.pack_into_tgz(settings.savetgz) dont_do_on_panic(cid) @@ -631,7 +769,7 @@ shutil.rmtree(self.name) logging.debug("Removed directory tree at %s" % self.name) elif settings.keep_tmpdir: - logging.debug("Keeping directory tree at %s" % self.name) + logging.debug("Keeping directory tree at %s" % self.name) def create_temp_tgz_file(self): """Return the path to a file to be used as a temporary tgz file""" @@ -640,16 +778,26 @@ (fd, temp_tgz) = create_temp_file() return temp_tgz + def remove_temp_tgz_file(self, temp_tgz): + """Remove the file that was used as a temporary tgz file""" + # Yes, remove_files() would work just as well, but putting it in + # the interface for Chroot allows the VirtServ hack to work. + remove_files([temp_tgz]) + def pack_into_tgz(self, result): """Tar and compress all files in the chroot.""" + self.run(["apt-get", "clean"]) logging.debug("Saving %s to %s." % (self.name, result)) - run(['tar', '--exclude', './proc/*', '-czf', result, '-C', self.name, './']) + run(['tar', '-czf', result, '--one-file-system', '--exclude', 'tmp/scripts', '-C', self.name, './']) def unpack_from_tgz(self, tarball): """Unpack a tarball to a chroot.""" logging.debug("Unpacking %s into %s" % (tarball, self.name)) - run(["tar", "-C", self.name, "-zxf", tarball]) + prefix = [] + if settings.eatmydata and os.path.isfile('/usr/bin/eatmydata'): + prefix.append('eatmydata') + run(prefix + ["tar", "-C", self.name, "-zxf", tarball]) def setup_from_lvm(self, lvm_volume): """Create a chroot by creating an LVM snapshot.""" @@ -664,7 +812,11 @@ run(['mount', self.lvm_snapshot, self.name]) def run(self, command, ignore_errors=False): - return run(["chroot", self.name] + command, + prefix = [] + if settings.eatmydata and os.path.isfile(os.path.join(self.name, + 'usr/bin/eatmydata')): + prefix.append('eatmydata') + return run(["chroot", self.name] + prefix + command, ignore_errors=ignore_errors) def create_apt_sources(self, distro): @@ -673,11 +825,11 @@ for mirror, components in settings.debian_mirrors: lines.append("deb %s %s %s\n" % (mirror, distro, " ".join(components))) - create_file(os.path.join(self.name, "etc/apt/sources.list"), + create_file(self.relative("etc/apt/sources.list"), "".join(lines)) def create_apt_conf(self): - """Create /etc/apt/apt.conf inside the chroot.""" + """Create /etc/apt/apt.conf.d/piuparts inside the chroot.""" lines = [ 'APT::Get::Assume-Yes "yes";\n', 'APT::Install-Recommends "0";\n', @@ -704,7 +856,7 @@ if settings.dpkg_force_confdef: lines.append('Dpkg::Options {"--force-confdef";};\n') - create_file(self.relative("etc/apt/apt.conf"), + create_file(self.relative("etc/apt/apt.conf.d/piuparts"), "".join(lines)) def create_dpkg_conf(self): @@ -716,24 +868,33 @@ lines.append('force-confdef\n') logging.info("Warning: dpkg has been configured to use the force-confdef option. This will hide problems, see #466118.") if lines: + if not os.path.exists(self.relative("etc/dpkg/dpkg.cfg.d")): + os.mkdir(self.relative("etc/dpkg/dpkg.cfg.d")) create_file(self.relative("etc/dpkg/dpkg.cfg.d/piuparts"), "".join(lines)) def create_policy_rc_d(self): """Create a policy-rc.d that prevents daemons from running.""" - full_name = os.path.join(self.name, "usr/sbin/policy-rc.d") + full_name = self.relative("usr/sbin/policy-rc.d") create_file(full_name, "#!/bin/sh\nexit 101\n") - os.chmod(full_name, 0777) - logging.debug("Created policy-rc.d and chmodded it.") + os.chmod(full_name, 0777) + logging.debug("Created policy-rc.d and chmodded it.") def setup_minimal_chroot(self): """Set up a minimal Debian system in a chroot.""" logging.debug("Setting up minimal chroot for %s at %s." % (settings.debian_distros[0], self.name)) + prefix = [] + if settings.eatmydata and os.path.isfile('/usr/bin/eatmydata'): + prefix.append('eatmydata') if settings.do_not_verify_signatures: logging.info("Warning: not using --keyring option when running debootstrap!") - run(["debootstrap", "--variant=minbase", settings.keyringoption, settings.debian_distros[0], - self.name, settings.debian_mirrors[0][0]]) + options = [settings.keyringoption] + if settings.eatmydata: + options.append('--include=eatmydata') + options.append('--components=%s' % ','.join(settings.debian_mirrors[0][1])) + run(prefix + ["debootstrap", "--variant=minbase"] + options + + [settings.debian_distros[0], self.name, settings.debian_mirrors[0][0]]) def minimize(self): """Minimize a chroot by removing (almost all) unnecessary packages""" @@ -746,6 +907,7 @@ def configure_chroot(self): """Configure a chroot according to current settings""" + os.environ["PIUPARTS_DISTRIBUTION"] = settings.debian_distros[0] if not settings.keep_sources_list: self.create_apt_sources(settings.debian_distros[0]) self.create_apt_conf() @@ -760,37 +922,57 @@ """Upgrade a chroot installation to each successive distro.""" for distro in distros: logging.debug("Upgrading %s to %s" % (self.name, distro)) + os.environ["PIUPARTS_DISTRIBUTION_NEXT"] = distro self.create_apt_sources(distro) - # Run custom scripts before upgrade - if settings.scriptsdir is not None: - self.run_scripts("pre_distupgrade") + # Run custom scripts before upgrade + self.run_scripts("pre_distupgrade") self.run(["apt-get", "update"]) self.run(["apt-get", "-yf", "dist-upgrade"]) + os.environ["PIUPARTS_DISTRIBUTION_PREV"] = os.environ["PIUPARTS_DISTRIBUTION"] + os.environ["PIUPARTS_DISTRIBUTION"] = distro # Sometimes dist-upgrade won't upgrade the packages we want # to test because the new version depends on a newer library, # and installing that would require removing the old version # of the library, and we've told apt-get not to remove # packages. So, we force the installation like this. - self.install_packages_by_name(packages) + if packages: + known_packages = self.get_known_packages(packages + settings.extra_old_packages) + self.install_packages_by_name(known_packages) # Run custom scripts after upgrade - if settings.scriptsdir is not None: - self.run_scripts("post_distupgrade") + self.run_scripts("post_distupgrade") self.check_for_no_processes() - - def apt_get_knows(self, package_names): - """Does apt-get (or apt-cache) know about a set of packages?""" - for name in package_names: + def apt_get_knows(self, packages): + return self.get_known_packages(packages) + + def get_known_packages(self, packages): + """Does apt-get (or apt-cache) know about a set of packages?""" + known_packages = [] + new_packages = [] + for name in packages: (status, output) = self.run(["apt-cache", "show", name], ignore_errors=True) - if status != 0: - return False - return True + # apt-cache reports status for some virtual packages and packages + # in status config-files-remaining state without installation + # candidate -- but only real packages have Filename/MD5sum/SHA* + if status != 0 or re.search(r'^(Filename|MD5sum|SHA1|SHA256):', output, re.M) is None: + new_packages.append(name) + else: + known_packages.append(name) + if not known_packages: + logging.info("apt-cache does not know about any of the requested packages") + else: + logging.info("apt-cache knows about the following packages: " + + ", ".join(known_packages)) + if new_packages: + logging.info("the following packages are not in the archive: " + + ", ".join(new_packages)) + return known_packages def copy_files(self, source_names, target_name): """Copy files in 'source_name' to file/dir 'target_name', relative to the root of the chroot.""" - target_name = os.path.join(self.name, target_name) + target_name = self.relative(target_name) logging.debug("Copying %s to %s" % (", ".join(source_names), target_name)) for source_name in source_names: @@ -800,7 +982,7 @@ logging.error("Error copying %s to %s: %s" % (source_name, target_name, detail)) panic() - + def list_installed_files (self, pre_info, post_info): """List the new files installed, removed and modified between two dir trees. Actually, it is a nice output of the funcion diff_meta_dat.""" @@ -810,7 +992,7 @@ if new: logging.debug("New installed files on system:\n" + file_list(new, file_owners)) else: - logging.debug("The package did not install any new file.\n") + logging.debug("The package did not install any new file.\n") if removed: logging.debug("The following files have disappeared:\n" + @@ -820,17 +1002,16 @@ logging.debug("The following files have been modified:\n" + file_list(modified, file_owners)) else: - logging.debug("The package did not modify any file.\n") + logging.debug("The package did not modify any file.\n") - def install_package_files(self, filenames): - if filenames: - self.copy_files(filenames, "tmp") - tmp_files = [os.path.basename(a) for a in filenames] + def install_package_files(self, package_files, packages = None): + if package_files: + self.copy_files(package_files, "tmp") + tmp_files = [os.path.basename(a) for a in package_files] tmp_files = [os.path.join("tmp", name) for name in tmp_files] - if settings.scriptsdir is not None: - self.run_scripts("pre_install") + self.run_scripts("pre_install") if settings.list_installed_files: pre_info = self.save_meta_data() @@ -847,12 +1028,9 @@ logging.info ("Installation of %s ok", tmp_files) - if settings.scriptsdir is not None: - self.run_scripts("post_install") + self.run_scripts("post_install") - self.run(["apt-get", "clean"]) - remove_files([os.path.join(self.name, name) - for name in tmp_files]) + remove_files([self.relative(name) for name in tmp_files]) def get_selections(self): """Get current package selections in a chroot.""" @@ -863,17 +1041,36 @@ vdict[name] = status return vdict - def remove_or_purge(self, operation, packages): - """Remove or purge packages in a chroot.""" - for name in packages: - self.run(["dpkg", "--" + operation, name], ignore_errors=True) - self.run(["dpkg", "--remove", "--pending"], ignore_errors=True) + def get_diversions(self): + """Get current dpkg-divert --list in a chroot.""" + if not settings.check_broken_diversions: + return + (status, output) = self.run(["dpkg-divert", "--list"]) + return output.split("\n") + def get_modified_diversions(self, pre_install_diversions, post_install_diversions = None): + """Check that diversions in chroot are identical (though potentially reordered).""" + if post_install_diversions is None: + post_install_diversions = self.get_diversions() + removed = [ln for ln in pre_install_diversions if not ln in post_install_diversions] + added = [ln for ln in post_install_diversions if not ln in pre_install_diversions] + return (removed, added) + + def remove_packages(self, packages): + """Remove packages in a chroot.""" + if packages: + self.run(["apt-get", "remove"] + packages, ignore_errors=True) + + def purge_packages(self, packages): + """Purge packages in a chroot.""" + if packages: + self.run(["dpkg", "--purge"] + packages, ignore_errors=True) - def restore_selections(self, changes, packages): - """Restore package selections in a chroot by applying 'changes'. - 'changes' is a return value from diff_selections.""" - + def restore_selections(self, selections, packages): + """Restore package selections in a chroot to the state in + 'selections'.""" + + changes = diff_selections(self, selections) deps = {} nondeps = {} for name, state in changes.iteritems(): @@ -881,7 +1078,7 @@ nondeps[name] = state else: deps[name] = state - + deps_to_remove = [name for name, state in deps.iteritems() if state == "remove"] deps_to_purge = [name for name, state in deps.iteritems() @@ -890,56 +1087,54 @@ if state == "remove"] nondeps_to_purge = [name for name, state in nondeps.iteritems() if state == "purge"] - + deps_to_install = [name for name, state in deps.iteritems() + if state == "install"] + # Run custom scripts before removing all packages. - if settings.scriptsdir is not None: - self.run_scripts("pre_remove") + self.run_scripts("pre_remove") # First remove all packages. - self.remove_or_purge("remove", deps_to_remove + deps_to_purge + - nondeps_to_remove + nondeps_to_purge) + self.remove_packages(deps_to_remove + deps_to_purge + + nondeps_to_remove + nondeps_to_purge) # Run custom scripts after removing all packages. - if settings.scriptsdir is not None: - self.run_scripts("post_remove") + self.run_scripts("post_remove") + + # Then reinstall missing packages. + if deps_to_install: + self.install_packages_by_name(deps_to_install) if not settings.skip_cronfiles_test: cronfiles, cronfiles_list = self.check_if_cronfiles(packages) - + if not settings.skip_cronfiles_test and cronfiles: self.check_output_cronfiles(cronfiles_list) if not settings.skip_logrotatefiles_test: logrotatefiles, logrotatefiles_list = self.check_if_logrotatefiles(packages) - + if not settings.skip_logrotatefiles_test and logrotatefiles: + installed = self.install_logrotate() self.check_output_logrotatefiles(logrotatefiles_list) + self.purge_packages(installed) # Then purge all packages being depended on. - self.remove_or_purge("purge", deps_to_purge) + self.purge_packages(deps_to_purge) # Finally, purge actual packages. - self.remove_or_purge("purge", nondeps_to_purge) - - # remove logrotate and it's depends - # (this is a fix for #602409 introduced by #566597 - # - search for the latter bug number in this file) - # XXX: another crude hack: ^^^ - if not settings.skip_logrotatefiles_test: - self.remove_or_purge("remove", ["adduser", "cron", "libpopt0", "logrotate"]) - self.remove_or_purge("purge", ["adduser", "cron", "libpopt0", "logrotate"]) - self.run(["apt-get", "clean"]) + self.purge_packages(nondeps_to_purge) # Run custom scripts after purge all packages. - if settings.scriptsdir is not None: - self.run_scripts("post_purge") + self.run_scripts("post_purge") # Now do a final run to see that everything worked. self.run(["dpkg", "--purge", "--pending"]) self.run(["dpkg", "--remove", "--pending"]) + self.run(["apt-get", "clean"]) def save_meta_data(self): """Return the filesystem meta data for all objects in the chroot.""" - root = os.path.join(self.name, ".") + self.run(["apt-get", "clean"]) + root = self.relative(".") vdict = {} proc = os.path.join(root, "proc") for dirpath, dirnames, filenames in os.walk(root): @@ -980,16 +1175,18 @@ def install_packages_by_name(self, packages): if packages: - if settings.scriptsdir is not None: - self.run_scripts("pre_install") + self.run_scripts("pre_install") - if settings.list_installed_files: + if settings.list_installed_files: pre_info = self.save_meta_data() self.run(["apt-get", "-y", "install"] + packages) self.list_installed_files (pre_info, self.save_meta_data()) else: self.run(["apt-get", "-y", "install"] + packages) + self.run_scripts("post_install") + + def check_for_no_processes(self): """Check there are no processes running inside the chroot.""" (status, output) = run(["lsof", "-w", "+D", self.name], ignore_errors=True) @@ -997,6 +1194,21 @@ if count > 0: logging.error("FAIL: Processes are running inside chroot:\n%s" % indent_string(output)) + for signo in [ 15, 9 ]: + p = subprocess.Popen(["lsof", "-t", "+D", self.name], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + stdout, _ = p.communicate() + if stdout: + pidlist = [int(pidstr) for pidstr in stdout.split("\n") if len(pidstr)] + for pid in pidlist: + if pid > 0: + try: + if signo == 15: + os.kill(pid, SIGTERM) + else: + os.kill(pid, SIGKILL) + except OSError: + pass panic() @@ -1060,7 +1272,7 @@ panic() else: logging.debug("No broken symlinks as far as we can find.") - + def check_if_cronfiles(self, packages): """Check if the packages have cron files under /etc/cron.d and in case positive, it returns the list of files. """ @@ -1071,7 +1283,7 @@ for p in packages: basename = p + ".list" - if not os.path.exists(os.path.join(vdir,basename)): + if not os.path.exists(os.path.join(vdir,basename)): continue f = file(os.path.join(vdir,basename), "r") @@ -1093,7 +1305,7 @@ def check_output_cronfiles (self, list): """Check if a given list of cronfiles has any output. Executes - cron file as cron would do (except for SHELL)""" + cron file as cron would do (except for SHELL)""" failed = False for vfile in list: @@ -1118,7 +1330,7 @@ for p in packages: basename = p + ".list" - if not os.path.exists(os.path.join(vdir,basename)): + if not os.path.exists(os.path.join(vdir,basename)): continue f = file(os.path.join(vdir,basename), "r") @@ -1134,13 +1346,18 @@ return has_logrotatefiles, vlist + def install_logrotate(self): + """Install logrotate for check_output_logrotatefiles, and return the + list of packages that were installed""" + old_selections = self.get_selections() + self.run(['apt-get', 'install', '-y', 'logrotate']) + diff = diff_selections(self, old_selections) + return diff.keys() + def check_output_logrotatefiles (self, list): """Check if a given list of logrotatefiles has any output. Executes logrotate file as logrotate would do from cron (except for SHELL)""" failed = False - # XXX That's a crude hack (to fix #602409). Can't we define a set of needed packages differently? - # It also introduces the need for hack to fix #602409 in piuparts.py - (a,b) = self.run(['apt-get','install', '-y', 'logrotate']) for vfile in list: if not os.path.exists(self.relative(vfile.strip("/"))): @@ -1157,6 +1374,8 @@ def run_scripts (self, step): """ Run custom scripts to given step post-install|remove|purge""" + if not settings.scriptsdirs: + return logging.info("Running scripts "+ step) basepath = self.relative("tmp/scripts/") if not os.path.exists(basepath): @@ -1248,6 +1467,9 @@ # adt-virt revert def create_temp_tgz_file(self): return self + def remove_temp_tgz_file(self, tgz): + if tgz is not self: self._fail('removing a tgz not supported') + # FIXME: anything else to do here? def pack_into_tgz(self, tgz): if tgz is not self: self._fail('packing into tgz not supported') if not 'revert' in self._caps: self._fail('testbed cannot revert') @@ -1492,13 +1714,11 @@ for name1, data1 in removed[:]: m = pat1.search(name1) if m: - pat2 = re.compile(r"^" + m.group(1) + r"[SK][0-9]{2}" + m.group(2) + -r"$") + pat2 = re.compile(r"^" + m.group(1) + r"[SK][0-9]{2}" + m.group(2) + r"$") for name2, data2 in new[:]: m = pat2.search(name2) if m: - logging.debug("File was renamed: %s\t=> %s" % (name1, -name2)) + logging.debug("File was renamed: %s\t=> %s" % (name1, name2)) removed.remove((name1, data1)) new.remove((name2, data2)) # this is again special casing due to the behaviour of a single package :( @@ -1516,8 +1736,8 @@ vlist.append(" %s\t" % name) if name in file_owners: vlist.append(" owned by: %s\n" % ", ".join(file_owners[name])) - else: - vlist.append(" not owned\n") + else: + vlist.append(" not owned\n") return "".join(vlist) @@ -1555,15 +1775,18 @@ if name not in selections: changes[name] = "purge" elif selections[name] != current[name] and \ - selections[name] == "purge": + selections[name] in ["purge", "install"]: changes[name] = selections[name] + for name, value in selections.iteritems(): + if name not in current: + changes[name] = "install" return changes -def get_package_names_from_package_files(filenames): +def get_package_names_from_package_files(package_files): """Return list of package names given list of package file names.""" vlist = [] - for filename in filenames: + for filename in package_files: (status, output) = run(["dpkg", "--info", filename]) for line in [line.lstrip() for line in output.split("\n")]: if line[:len("Package:")] == "Package:": @@ -1613,7 +1836,7 @@ def check_results(chroot, root_info, file_owners, deps_info=None): """Check that current chroot state matches 'root_info'. - + If settings.warn_on_others is True and deps_info is not None, then only print a warning rather than failing if the current chroot contains files that are in deps_info but not in root_info. (In this case, deps_info @@ -1622,6 +1845,18 @@ installed.) """ + ok = True + if settings.check_broken_diversions: + (removed, added) = chroot.get_modified_diversions(chroot.pre_install_diversions) + if added: + logging.error("FAIL: Installed diversions (dpkg-divert) not removed by purge:\n%s" % + indent_string("\n".join(added))) + ok = False + if removed: + logging.error("FAIL: Existing diversions (dpkg-divert) removed/modified:\n%s" % + indent_string("\n".join(removed))) + ok = False + current_info = chroot.save_meta_data() if settings.warn_on_others and deps_info is not None: (new, removed, modified) = diff_meta_data(root_info, current_info) @@ -1635,7 +1870,6 @@ else: (new, removed, modified) = diff_meta_data(root_info, current_info) - ok = True if new: if settings.warn_on_leftovers_after_purge: logging.info("Warning: Package purging left files on system:\n" + @@ -1676,20 +1910,24 @@ return ok -def install_purge_test(chroot, root_info, selections, package_list, packages): +def install_purge_test(chroot, root_info, selections, package_files, packages): """Do an install-purge test. Return True if successful, False if not. Assume 'root' is a directory already populated with a working chroot, with packages in states given by 'selections'.""" + os.environ["PIUPARTS_TEST"] = "install" + chroot.run_scripts("pre_test") + # Install packages into the chroot. + os.environ["PIUPARTS_PHASE"] = "install" if settings.warn_on_others: # Create a metapackage with dependencies from the given packages - if package_list: + if package_files: control_infos = [] # We were given package files, so let's get the Depends and # Conflicts directly from the .debs - for deb in package_list: + for deb in package_files: returncode, output = run(["dpkg", "-f", deb]) control = deb822.Deb822(output) control_infos.append(control) @@ -1700,7 +1938,7 @@ apt_cache_args.extend(packages) returncode, output = chroot.run(apt_cache_args) control_infos = deb822.Deb822.iter_paragraphs(output.splitlines()) - + depends = [] conflicts = [] for control in control_infos: @@ -1712,12 +1950,12 @@ all_conflicts = ", ".join(conflicts) metapackage = make_metapackage("piuparts-depends-dummy", all_depends, all_conflicts) - + # Install the metapackage chroot.install_package_files([metapackage]) # Now remove it metapackagename = os.path.basename(metapackage)[:-4] - chroot.remove_or_purge("purge", [metapackagename]) + chroot.purge_packages([metapackagename]) shutil.rmtree(os.path.dirname(metapackage)) # Save the file ownership information so we can tell which @@ -1727,11 +1965,10 @@ else: deps_info = None - if package_list: - chroot.install_package_files(package_list) + if package_files: + chroot.install_package_files(package_files, packages) else: chroot.install_packages_by_name(packages) - chroot.run(["apt-get", "clean"]) chroot.check_for_no_processes() @@ -1740,36 +1977,40 @@ file_owners = chroot.get_files_owned_by_packages() # Remove all packages from the chroot that weren't there initially. - changes = diff_selections(chroot, selections) - chroot.restore_selections(changes, packages) - + chroot.restore_selections(selections, packages) + + chroot.check_for_no_processes() chroot.check_for_broken_symlinks() return check_results(chroot, root_info, file_owners, deps_info=deps_info) -def install_upgrade_test(chroot, root_info, selections, package_list, package_names): - """Install package via apt-get, then upgrade from package files. +def install_upgrade_test(chroot, root_info, selections, package_files, packages, old_packages): + """Install old_packages via apt-get, then upgrade from package files. Return True if successful, False if not.""" + os.environ["PIUPARTS_TEST"] = "upgrade" + chroot.run_scripts("pre_test") + # First install via apt-get. - chroot.install_packages_by_name(package_names) - - if settings.scriptsdir is not None: - chroot.run_scripts("pre_upgrade") + os.environ["PIUPARTS_PHASE"] = "install" + chroot.install_packages_by_name(old_packages) + chroot.check_for_no_processes() chroot.check_for_broken_symlinks() # Then from the package files. - chroot.install_package_files(package_list) - + os.environ["PIUPARTS_PHASE"] = "upgrade" + chroot.install_package_files(package_files, packages) + + chroot.check_for_no_processes() + chroot.check_for_broken_symlinks() + file_owners = chroot.get_files_owned_by_packages() - # Remove all packages from the chroot that weren't there - # initially. - changes = diff_selections(chroot, selections) - chroot.restore_selections(changes, package_names) - + # Remove all packages from the chroot that weren't there initially. + chroot.restore_selections(selections, packages) + chroot.check_for_no_processes() chroot.check_for_broken_symlinks() @@ -1793,7 +2034,7 @@ return root_info, selections -def install_and_upgrade_between_distros(filenames, packages): +def install_and_upgrade_between_distros(package_files, packages): """Install package and upgrade it between distributions, then remove. Return True if successful, False if not.""" @@ -1819,70 +2060,82 @@ # a reasonable default behaviour for distro upgrade tests, which are not # done by default anyway. + os.environ["PIUPARTS_TEST"] = "distupgrade" + chroot = get_chroot() chroot.create() cid = do_on_panic(chroot.remove) - if settings.basetgz: - root_tgz = settings.basetgz - else: - root_tgz = chroot.create_temp_tgz_file() - chroot.pack_into_tgz(root_tgz) - if settings.end_meta: # load root_info and selections root_info, selections = load_meta_data(settings.end_meta) + chroot.pre_install_diversions = [] # FIXME: diversion info needs to be restored else: + if not settings.basetgz: + temp_tgz = chroot.create_temp_tgz_file() + # FIXME: on panic remove temp_tgz + chroot.pack_into_tgz(temp_tgz) + chroot.upgrade_to_distros(settings.debian_distros[1:], []) - chroot.run(["apt-get", "clean"]) + + chroot.check_for_no_processes() # set root_info and selections root_info = chroot.save_meta_data() selections = chroot.get_selections() - + diversions = chroot.get_diversions() + if settings.save_end_meta: # save root_info and selections save_meta_data(settings.save_end_meta, root_info, selections) - + # FIXME: diversion info needs to be stored + chroot.remove() dont_do_on_panic(cid) + + # leave indication in logfile why we do what we do + logging.info("Notice: package selections and meta data from target disto saved, now starting over from source distro. See the description of --save-end-meta and --end-meta to learn why this is neccessary and how to possibly avoid it.") + chroot = get_chroot() - chroot.create() + if settings.basetgz: + chroot.create() + else: + chroot.create(temp_tgz) + chroot.remove_temp_tgz_file(temp_tgz) + chroot.pre_install_diversions = diversions cid = do_on_panic(chroot.remove) - # leave indication in logfile why we do what we do - logging.info("Notice: package selections and meta data from target disto saved, now starting over from source distro. See the description of --save-end-meta and --end-meta to learn why this is neccessary and how to possibly avoid it.") - chroot.check_for_no_processes() - chroot.run(["apt-get", "update"]) - chroot.install_packages_by_name(packages) + chroot.run_scripts("pre_test") - if settings.scriptsdir is not None: - chroot.run_scripts("pre_upgrade") + os.environ["PIUPARTS_PHASE"] = "install" + + known_packages = chroot.get_known_packages(packages + settings.extra_old_packages) + chroot.install_packages_by_name(known_packages) chroot.check_for_no_processes() + os.environ["PIUPARTS_PHASE"] = "distupgrade" + chroot.upgrade_to_distros(settings.debian_distros[1:], packages) chroot.check_for_no_processes() - chroot.install_package_files(filenames) - chroot.run(["apt-get", "clean"]) - + os.environ["PIUPARTS_PHASE"] = "upgrade" + + chroot.install_package_files(package_files, packages) + chroot.check_for_no_processes() file_owners = chroot.get_files_owned_by_packages() # use root_info and selections - changes = diff_selections(chroot, selections) - chroot.restore_selections(changes, packages) + chroot.restore_selections(selections, packages) result = check_results(chroot, root_info, file_owners) chroot.check_for_no_processes() - - if root_tgz != settings.basetgz: - remove_files([root_tgz]) + chroot.remove() dont_do_on_panic(cid) @@ -1925,11 +2178,11 @@ def parse_command_line(): """Parse the command line, change global settings, return non-options.""" - + parser = optparse.OptionParser(usage="%prog [options] package ...", version="piuparts %s" % VERSION) - - + + parser.add_option("-a", "--apt", action="store_true", default=False, help="Command line arguments are package names " + "to be installed via apt.") @@ -1938,7 +2191,7 @@ metavar='CMDLINE', default=None, help="Use CMDLINE via autopkgtest (adt-virt-*)" " protocol instead of managing a chroot.") - + parser.add_option("-b", "--basetgz", metavar="TARBALL", help="Use TARBALL as the contents of the initial " + "chroot, instead of building a new one with " + @@ -1947,7 +2200,7 @@ parser.add_option("--bindmount", action="append", metavar="DIR", default=[], help="Directory to be bind-mounted inside the chroot.") - + parser.add_option("-d", "--distribution", action="append", metavar="NAME", help="Which Debian distribution to use: a code name " + "(for example lenny, squeeze, sid) or experimental. The " + @@ -1956,21 +2209,26 @@ parser.add_option("-D", "--defaults", action="store", help="Choose which set of defaults to use " "(debian/ubuntu).") - + parser.add_option("--debfoster-options", default="-o MaxPriority=important -o UseRecommends=no -f -n apt debfoster", help="Run debfoster with different parameters (default: -o MaxPriority=important -o UseRecommends=no -f -n apt debfoster).") + parser.add_option("--no-eatmydata", + default=False, + action='store_true', + help="Default is to use libeatmydata in the chroot") + parser.add_option("--dpkg-noforce-unsafe-io", default=False, action='store_true', - help="Default is to run dpkg with --force-unsafe-io option, which causes dpkg to skip certain file system syncs known to cause substantial performance degradation on some filesystems. This option turns that off and dpkg will use safe I/O operations.") + help="Default is to run dpkg with --force-unsafe-io option, which causes dpkg to skip certain file system syncs known to cause substantial performance degradation on some filesystems. This option turns that off and dpkg will use safe I/O operations.") parser.add_option("--dpkg-force-confdef", default=False, action='store_true', - help="Make dpkg use --force-confdef, which lets dpkg always choose the default action when a modified conffile is found. This option will make piuparts ignore errors it was designed to report and therefore should only be used to hide problems in depending packages. (See #466118.)") - + help="Make dpkg use --force-confdef, which lets dpkg always choose the default action when a modified conffile is found. This option will make piuparts ignore errors it was designed to report and therefore should only be used to hide problems in depending packages. (See #466118.)") + parser.add_option("--do-not-verify-signatures", default=False, action='store_true', help="Do not verify signatures from the Release files when running debootstrap.") @@ -1979,13 +2237,13 @@ default=[], help="Add FILENAME to list of filenames to be " + "ignored when comparing changes to chroot.") - + parser.add_option("-I", "--ignore-regex", action="append", metavar="REGEX", default=[], help="Add REGEX to list of Perl compatible regular " + "expressions for filenames to be " + "ignored when comparing changes to chroot.") - + parser.add_option("-k", "--keep-tmpdir", action="store_true", default=False, help="Don't remove the temporary directory for the " + @@ -1994,7 +2252,7 @@ parser.add_option("-K", "--keyring", metavar="FILE", default = "/usr/share/keyrings/ubuntu-archive-keyring.gpg", help="Use FILE as the keyring to use with debootstrap when creating chroots.") - + parser.add_option("--keep-sources-list", action="store_true", default=False, help="Don't modify the chroot's " + @@ -2008,7 +2266,7 @@ parser.add_option("--list-installed-files", action="store_true", default=False, help="List files added to the chroot after the " + - "installation of the package.") + "installation of the package.") parser.add_option("--lvm-volume", metavar="LVM-VOL", action="store", help="Use LVM-VOL as source for the chroot, instead of building " + @@ -2018,16 +2276,20 @@ parser.add_option("--lvm-snapshot-size", metavar="SNAPSHOT-SIZE", action="store", default="1G", help="Use SNAPSHOT-SIZE as snapshot size when creating " + "a new LVM snapshot (default: 1G)") - + parser.add_option("-m", "--mirror", action="append", metavar="URL", default=[], help="Which Debian mirror to use.") - + + parser.add_option("--no-diversions", action="store_true", + default=False, + help="Don't check for broken diversions.") + parser.add_option("-n", "--no-ignores", action="callback", callback=forget_ignores, help="Forget all ignores set so far, including " + "built-in ones.") - + parser.add_option("-N", "--no-symlinks", action="store_true", default=False, help="Don't check for broken symlinks.") @@ -2035,12 +2297,18 @@ parser.add_option("--no-upgrade-test", action="store_true", default=False, help="Skip testing the upgrade from an existing version " + - "in the archive.") - + "in the archive.") + parser.add_option("--no-install-purge-test", action="store_true", default=False, help="Skip install and purge test.") + parser.add_option("--extra-old-packages", + action="append", default=[], + help="Install these additional packages along with the old packages from the archive. " + + "Useful to test Conflicts/Replaces of packages that will disappear during the update. " + + "Takes a comma separated list of package names and can be given multiple times.") + parser.add_option("-p", "--pbuilder", action="callback", callback=set_basetgz_to_pbuilder, help="Use /var/cache/pbuilder/base.tgz as the base " + @@ -2049,13 +2317,13 @@ parser.add_option("--pedantic-purge-test", action="store_true", default=False, help="Be pedantic when checking if a purged package leaves files behind. If this option is not set, files left in /tmp are ignored.") - + parser.add_option("-s", "--save", metavar="FILENAME", help="Save the chroot into FILENAME.") parser.add_option("-B", "--end-meta", metavar="FILE", help="Save chroot package selection and file meta data in FILE for later use. See the function install_and_upgrade_between_distros() in piuparts.py for defaults. Mostly useful for large scale distro upgrade tests.") - + parser.add_option("-S", "--save-end-meta", metavar="FILE", help="Load chroot package selection and file meta data from FILE. See the function install_and_upgrade_between_distros() in piuparts.py for defaults. Mostly useful for large scale distro upgrade tests.") @@ -2066,7 +2334,7 @@ parser.add_option("--skip-cronfiles-test", action="store_true", default=False, help="Skip testing the output from the cron files.") - + parser.add_option("--skip-logrotatefiles-test", action="store_true", default=False, help="Skip testing the output from the logrotate files.") @@ -2080,8 +2348,9 @@ help="Minimize chroot with debfoster. This used to be the default until #539142 was fixed.") parser.add_option("--scriptsdir", metavar="DIR", - help="Directory where are placed the custom scripts.") - + action="append", default=[], + help="Directory where are placed the custom scripts. Can be given multiple times.") + parser.add_option("-t", "--tmpdir", metavar="DIR", help="Use DIR for temporary storage. Default is " + "$TMPDIR or /tmp.") @@ -2110,7 +2379,7 @@ parser.add_option("--fail-on-broken-symlinks", action="store_true", default=False, help="Fail if broken symlinks are detected.") - + parser.add_option("--log-level", action="store",metavar='LEVEL', default="dump", help="Displays messages from LEVEL level, possible values are: error, info, dump, debug. The default is dump.") @@ -2131,9 +2400,12 @@ settings.keep_sources_list = opts.keep_sources_list settings.skip_minimize = opts.skip_minimize settings.minimize = opts.minimize + if settings.minimize: + settings.skip_minimize = False settings.list_installed_files = opts.list_installed_files settings.no_install_purge_test = opts.no_install_purge_test settings.no_upgrade_test = opts.no_upgrade_test + [settings.extra_old_packages.extend([i.strip() for i in csv.split(",")]) for csv in opts.extra_old_packages] settings.skip_cronfiles_test = opts.skip_cronfiles_test settings.skip_logrotatefiles_test = opts.skip_logrotatefiles_test settings.keyring = opts.keyring @@ -2147,19 +2419,21 @@ settings.pedantic_purge_test = opts.pedantic_purge_test if not settings.pedantic_purge_test: settings.ignored_patterns += settings.non_pedantic_ignore_patterns - + log_file_name = opts.log_file defaults = DefaultsFactory().new_defaults() - + settings.debian_mirrors = [parse_mirror_spec(x, defaults.get_components()) for x in opts.mirror] + settings.check_broken_diversions = not opts.no_diversions settings.check_broken_symlinks = not opts.no_symlinks settings.warn_broken_symlinks = not opts.fail_on_broken_symlinks settings.savetgz = opts.save settings.warn_on_others = opts.warn_on_others settings.warn_on_leftovers_after_purge = opts.warn_on_leftovers_after_purge settings.debfoster_options = opts.debfoster_options.split() + settings.eatmydata = not opts.no_eatmydata settings.dpkg_force_unsafe_io = not opts.dpkg_noforce_unsafe_io settings.dpkg_force_confdef = opts.dpkg_force_confdef @@ -2192,11 +2466,10 @@ else: settings.tmpdir = "/tmp" - if opts.scriptsdir is not None: - settings.scriptsdir = opts.scriptsdir - if not os.path.isdir(settings.scriptsdir): - logging.error("Scripts directory is not a directory: %s" % - settings.scriptsdir) + settings.scriptsdirs = opts.scriptsdir + for sdir in settings.scriptsdirs: + if not os.path.isdir(sdir): + logging.error("Scripts directory is not a directory: %s" % sdir) panic() if not settings.debian_distros: @@ -2223,7 +2496,7 @@ sys.exit(exitcode) return args - + def get_chroot(): if settings.adt_virt is None: return Chroot() @@ -2234,9 +2507,10 @@ # Find the names of packages. if settings.args_are_package_files: packages = get_package_names_from_package_files(package_list) + package_files = package_list else: packages = package_list - package_list = [] + package_files = [] if len(settings.debian_distros) == 1: chroot = get_chroot() @@ -2244,11 +2518,12 @@ cid = do_on_panic(chroot.remove) root_info = chroot.save_meta_data() + chroot.pre_install_diversions = chroot.get_diversions() selections = chroot.get_selections() if not settings.no_install_purge_test: if not install_purge_test(chroot, root_info, selections, - package_list, packages): + package_files, packages): logging.error("FAIL: Installation and purging test.") panic() logging.info("PASS: Installation and purging test.") @@ -2256,19 +2531,23 @@ if not settings.no_upgrade_test: if not settings.args_are_package_files: logging.info("Can't test upgrades: -a or --apt option used.") - elif not chroot.apt_get_knows(packages): - logging.info("Can't test upgrade: packages not known by apt-get.") - elif install_upgrade_test(chroot, root_info, selections, package_list, - packages): - logging.info("PASS: Installation, upgrade and purging tests.") else: - logging.error("FAIL: Installation, upgrade and purging tests.") - panic() - + packages_to_query = packages[:] + packages_to_query.extend(settings.extra_old_packages) + known_packages = chroot.get_known_packages(packages_to_query) + if not known_packages: + logging.info("Can't test upgrade: packages not known by apt-get.") + elif install_upgrade_test(chroot, root_info, selections, package_files, + packages, known_packages): + logging.info("PASS: Installation, upgrade and purging tests.") + else: + logging.error("FAIL: Installation, upgrade and purging tests.") + panic() + chroot.remove() dont_do_on_panic(cid) else: - if install_and_upgrade_between_distros(package_list, packages): + if install_and_upgrade_between_distros(package_files, packages): logging.info("PASS: Upgrading between Debian distributions.") else: logging.error("FAIL: Upgrading between Debian distributions.") @@ -2335,3 +2614,5 @@ print '' print 'Piuparts interrupted by the user, exiting...' sys.exit(1) + +# vi:set et ts=4 sw=4 : diff -Nru piuparts-0.41ubuntu2/piuparts-report.py piuparts-0.42ubuntu1/piuparts-report.py --- piuparts-0.41ubuntu2/piuparts-report.py 2011-10-24 17:48:07.000000000 +0000 +++ piuparts-0.42ubuntu1/piuparts-report.py 2012-01-03 16:41:52.000000000 +0000 @@ -185,8 +185,11 @@