diff -Nru apt-cacher-ng-0.7.7/ChangeLog apt-cacher-ng-0.7.11/ChangeLog --- apt-cacher-ng-0.7.7/ChangeLog 2012-08-04 18:38:56.000000000 +0000 +++ apt-cacher-ng-0.7.11/ChangeLog 2012-12-16 18:56:41.000000000 +0000 @@ -1,3 +1,45 @@ +apt-cacher-ng (0.7.11) BETA; urgency=low + + * FIX: iterator out of bounds when looking for file item replacements + * Spelling fix: s/Threat/Treat/ + + -- + +apt-cacher-ng (0.7.10) BETA; urgency=low + + * Fixes accidential removal of .rpm files covered by XML indes of unknown + format in the cleanup routine + + -- Eduard Bloch Tue, 06 Nov 2012 23:23:38 +0100 + +apt-cacher-ng (0.7.9) BETA; urgency=low + + * Fixed downloads with the ForceManaged mode (too paranoid redirection + checks and not implemented correctly) + * Mirror database update. Removed http.debian.net from regular Debian + mirrors URL list again, there is some potential of involuntary creation of + endless redirection loops. Should better be used by administrators who + know what they are doing. + + -- Eduard Bloch Mon, 05 Nov 2012 23:16:58 +0100 + +apt-cacher-ng (0.7.8) BETA; urgency=low + + * FIX: overhaul of the http redirection handling, it all moved to the + download agent where it actually belongs to. Can now silently resume a + download if a mirror suddenly failes while download was running. + * FIX: partial rewrite of cache management code, getting rid of some + evolutionary created junk. Now possible to detect file descriptors that + got stuck because of some blocking client and setup a replacement program + path for other clients when they need a fresh version of the remote file. + * NEW: SIGUSR2 writes an overview of the cache state to the error log + * Database update, also includes http.debian.net now. + * Adding .html suffix to maintenance reports, for convenience. + * Added .drpm files to filename patterns + * Mirror database update + + -- Eduard Bloch Sun, 04 Nov 2012 20:38:02 +0100 + apt-cacher-ng (0.7.7) BETA; urgency=low * FIX: moved object cleanup activity to a dedicated thread, review and diff -Nru apt-cacher-ng-0.7.7/GNUmakefile apt-cacher-ng-0.7.11/GNUmakefile --- apt-cacher-ng-0.7.7/GNUmakefile 2012-04-16 23:12:15.000000000 +0000 +++ apt-cacher-ng-0.7.11/GNUmakefile 2012-11-05 22:24:59.000000000 +0000 @@ -140,6 +140,8 @@ perl dbgen/deburlgen.pl Archive-http < $(DBTMP)/dsnap > $(DBTMP)/dsnap.urls echo http://ftp.debian.org/debian/ >> $(DBTMP)/dsnap.urls echo http://cdn.debian.net/debian/ >> $(DBTMP)/dsnap.urls +# a way too experimental... echo http://http.debian.net/debian/ >> $(DBTMP)/dsnap.urls +# there is no pool... echo http://ftp-master.debian.org/newdists/ >> $(DBTMP)/dsnap.urls bash dbgen/ubuntuscan.sh conf/deb_mirrors $(DBTMP)/dsnap.urls $(DBTMP) "/dists" gzip -f -9 conf/deb_mirrors diff -Nru apt-cacher-ng-0.7.7/VERSION apt-cacher-ng-0.7.11/VERSION --- apt-cacher-ng-0.7.7/VERSION 2012-07-23 13:38:14.000000000 +0000 +++ apt-cacher-ng-0.7.11/VERSION 2012-12-16 18:56:41.000000000 +0000 @@ -1 +1 @@ -0.7.7 +0.7.11 diff -Nru apt-cacher-ng-0.7.7/conf/acng.conf apt-cacher-ng-0.7.11/conf/acng.conf --- apt-cacher-ng-0.7.7/conf/acng.conf 2012-07-23 19:39:51.000000000 +0000 +++ apt-cacher-ng-0.7.11/conf/acng.conf 2012-11-04 19:38:17.000000000 +0000 @@ -121,12 +121,13 @@ # Pigeonholing files with regular expressions (static/volatile). Can be # overriden here but not should not be done permanently because future update # of default settings would not be applied later. -# VfilePattern = (^|.*?/)(Index|Packages(\.gz|\.bz2|\.lzma|\.xz)?|InRelease|Release|Release\.gpg|Sources(\.gz|\.bz2|\.lzma|\.xz)?|release|index\.db-.*\.gz|Contents-[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|pkglist[^/]*\.bz2|rclist[^/]*\.bz2|/meta-release[^/]*|Translation[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|MD5SUMS|SHA1SUMS|((setup|setup-legacy)(\.ini|\.bz2|\.hint)(\.sig)?)|mirrors\.lst|repo(index|md)\.xml(\.asc|\.key)?|directory\.yast|products|content(\.asc|\.key)?|media|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz|info\.txt|license\.tar\.gz|license\.zip|.*\.db(\.tar\.gz)?|.*\.files\.tar\.gz|.*\.abs\.tar\.gz|metalink\?repo|.*prestodelta\.xml\.gz)$|/dists/.*/installer-[^/]+/[^0-9][^/]+/images/.* -# PfilePattern = .*(\.d?deb|\.rpm|\.dsc|\.tar(\.gz|\.bz2|\.lzma|\.xz)(\.gpg)?|\.diff(\.gz|\.bz2|\.lzma|\.xz)|\.jigdo|\.template|changelog|copyright|\.udeb|\.debdelta|\.diff/.*\.gz|(Devel)?ReleaseAnnouncement(\?.*)?|[a-f0-9]+-(susedata|updateinfo|primary|deltainfo).xml.gz|fonts/(final/)?[a-z]+32.exe(\?download.*)?|/dists/.*/installer-[^/]+/[0-9][^/]+/images/.*)$ +# VfilePattern = (^|.*/)(Index|Packages(\.gz|\.bz2|\.lzma|\.xz)?|InRelease|Release|Release\.gpg|Sources(\.gz|\.bz2|\.lzma|\.xz)?|release|index\.db-.*\.gz|Contents-[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|pkglist[^/]*\.bz2|rclist[^/]*\.bz2|meta-release[^/]*|Translation[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|MD5SUMS|SHA1SUMS|((setup|setup-legacy)(\.ini|\.bz2|\.hint)(\.sig)?)|mirrors\.lst|repo(index|md)\.xml(\.asc|\.key)?|directory\.yast|products|content(\.asc|\.key)?|media|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz|info\.txt|license\.tar\.gz|license\.zip|.*\.(db|files|abs)(\.tar(\.gz|\.bz2|\.lzma|\.xz))?|metalink\?repo|.*prestodelta\.xml\.gz|repodata/.*\.(xml|sqlite)(\.gz|\.bz2|\.lzma|\.xz))$|/dists/.*/installer-[^/]+/[^0-9][^/]+/images/.* +# PfilePattern = .*(\.d?deb|\.rpm|\.drpm|\.dsc|\.tar(\.gz|\.bz2|\.lzma|\.xz)(\.gpg)?|\.diff(\.gz|\.bz2|\.lzma|\.xz)|\.jigdo|\.template|changelog|copyright|\.udeb|\.debdelta|\.diff/.*\.gz|(Devel)?ReleaseAnnouncement(\?.*)?|[a-f0-9]+-(susedata|updateinfo|primary|deltainfo).xml.gz|fonts/(final/)?[a-z]+32.exe(\?download.*)?|/dists/.*/installer-[^/]+/[0-9][^/]+/images/.*)$ + # Whitelist for expiration, file types not to be removed even when being # unreferenced. Default: many parts from VfilePattern where no parent index # exists or might be unknown. -# WfilePattern = (^|.*?/)(Release|InRelease|Release\.gpg|(Packages|Sources)(\.gz|\.bz2|\.lzma|\.xz)?|Translation[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|MD5SUMS|SHA1SUMS|.*\.xml|.*\.db\.tar\.gz|.*\.files\.tar\.gz|.*\.abs\.tar\.gz|[a-z]+32.exe)$|/dists/.*/installer-.*/images/.* +#WfilePattern = (^|.*/)(Release|InRelease|Release\.gpg|(Packages|Sources)(\.gz|\.bz2|\.lzma|\.xz)?|Translation[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|MD5SUMS|SHA1SUMS|.*\.xml|.*\.(db|files|abs)(\.tar(\.gz|\.bz2|\.lzma|\.xz))?|[a-z]+32.exe)$|/dists/.*/installer-.*/images/.* # Higher modes only working with the debug version # Warning, writes a lot into apt-cacher.err logfile @@ -291,3 +292,14 @@ # Examples: # PassThroughPattern: private-ppa\.launchpad\.net:443$ # PassThroughPattern: .* # this would allow CONNECT to everything + +# It's possible that an evil client requests a volatile file but does not +# retrieve the response and keeps the connection effectively stuck over +# many hours, blocking the particular file for other download attempts (which +# leads to not reporting updates on server side to other users). The work +# around is the use of alternative file descriptors inside of apt-cacher-ng, +# however this might cost some extra download traffic due to worse cache usage. +# The ResponseFreezeDetectTime value specifies when a file descriptor in the +# mentioned state is to be considered defect and will require special handling. +# Default time is 500 seconds. +# ResponseFreezeDetectTime: 500 Binary files /tmp/U2Bd2QnZSs/apt-cacher-ng-0.7.7/conf/deb_mirrors.gz and /tmp/YGtW7qUsez/apt-cacher-ng-0.7.11/conf/deb_mirrors.gz differ Binary files /tmp/U2Bd2QnZSs/apt-cacher-ng-0.7.7/conf/debvol_mirrors.gz and /tmp/YGtW7qUsez/apt-cacher-ng-0.7.11/conf/debvol_mirrors.gz differ diff -Nru apt-cacher-ng-0.7.7/conf/epel_mirrors apt-cacher-ng-0.7.11/conf/epel_mirrors --- apt-cacher-ng-0.7.7/conf/epel_mirrors 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/conf/epel_mirrors 2012-11-05 22:24:59.000000000 +0000 @@ -1,5 +1,6 @@ http://archive.linux.duke.edu/pub/epel/ http://be.mirror.eurid.eu/epel/ +http://bigstorage.tren.cc/centos/epel/ http://dl.fedoraproject.org/pub/epel/ http://epel.blizoo.mk/epel/ http://epel.idrepo.or.id/epel/ @@ -18,7 +19,6 @@ http://fr2.rpmfind.net/linux/epel/ http://ftp-stud.hs-esslingen.de/pub/epel/ http://ftp.astral.ro/mirrors/fedora/pub/epel/ -http://ftp.byfly.by/pub/fedoraproject.org/epel/ http://ftp.cc.uoc.gr/pub/epel/ http://ftp.cica.es/epel/ http://ftp.crc.dk/fedora-epel/ @@ -38,55 +38,61 @@ http://ftp.ntua.gr/pub/linux/fedora-epel/ http://ftp.osuosl.org/pub/fedora-epel/ http://ftp.ps.pl/pub/Linux/fedora-epel/ -http://ftp.rediris.es/mirror/fedora-epel/ http://ftp.rhd.ru/pub/epel/ http://ftp.riken.jp/Linux/fedora/epel/ -http://ftp.sh.cvut.cz/MIRRORS/epel/ http://ftp.tlk-l.net/pub/mirrors/fedora-epel/ http://ftp.tsukuba.wide.ad.jp/Linux/fedora/epel/ http://ftp.uni-bayreuth.de/linux/fedora-epel/ +http://ftp.uni-kl.de/pub/linux/fedora-epel/ http://ftp.uni-koeln.de/mirrors/fedora/epel/ http://ftp.uninett.no/linux/epel/ -http://ftp.yz.yamagata-u.ac.jp/pub/linux/fedora/epel/ +http://ftp.upjs.sk/pub/mirrors/epel/ +http://kartolo.sby.datautama.net.id/EPEL/ http://kdeforge.unl.edu/mirrors/epel/ -http://less.cogeco.net/epel/ http://linux.mirrors.es.net/fedora-epel/ http://madura.idrepo.or.id/epel/ http://mir01.syntis.net/epel/ http://mirror-fpt-telecom.fpt.net/fedora/epel/ http://mirror.affix.me/epel/ http://mirror.ancl.hawaii.edu/linux/epel/ -http://mirror.as24220.net/pub/epel/ http://mirror.astpage.ru/epel/ http://mirror.awanti.com/epel/ http://mirror.beyondhosting.net/Fedora-Epel/ -http://mirror.bjtu.edu.cn/fedora-epel/ http://mirror.bytemark.co.uk/fedora/epel/ http://mirror.chpc.utah.edu/pub/epel/ +http://mirror.clarkson.edu/epel/ http://mirror.cogentco.com/pub/linux/epel/ http://mirror.csclub.uwaterloo.ca/fedora/epel/ http://mirror.datacenter.by/pub/fedoraproject.org/epel/ http://mirror.de.leaseweb.net/epel/ http://mirror.digmia.com/epel/ +http://mirror.duomenucentras.lt/epel/ +http://mirror.epel.delta-net.ge/epel/ http://mirror.fraunhofer.de/download.fedora.redhat.com/epel/ http://mirror.hiwaay.net/pub/epel/ http://mirror.hmc.edu/epel/ http://mirror.hosting90.cz/epel/ http://mirror.i3d.net/pub/fedora-epel/ http://mirror.ibcp.fr/pub/epel/ +http://mirror.imt-systems.com/epel/ http://mirror.itc.virginia.edu/fedora-epel/ http://mirror.its.dal.ca/pub/epel/ http://mirror.karneval.cz/pub/linux/fedora/epel/ +http://mirror.kiewel-online.ch/epel/ http://mirror.metrocast.net/fedora/epel/ http://mirror.neu.edu.cn/fedora/epel/ http://mirror.nexcess.net/epel/ http://mirror.nl.leaseweb.net/epel/ http://mirror.nsc.liu.se/fedora-epel/ +http://mirror.nus.edu.sg/Fedora/epel/ +http://mirror.nus.edu.sg/fedora/epel/ http://mirror.optus.net/epel/ http://mirror.overthewire.com.au/pub/epel/ http://mirror.pmf.kg.ac.rs/fedora/epel/ http://mirror.pnl.gov/epel/ +http://mirror.proserve.nl/fedora-epel/ http://mirror.seas.harvard.edu/epel/ +http://mirror.slu.cz/epel/ http://mirror.steadfast.net/epel/ http://mirror.switch.ch/ftp/mirror/epel/ http://mirror.symnds.com/distributions/fedora-epel/ @@ -108,7 +114,9 @@ http://mirrors.dmacc.net/pub/epel/ http://mirrors.einstein.yu.edu/epel/ http://mirrors.ircam.fr/pub/fedora/epel/ +http://mirrors.ispros.com.bd/fedora-epel/ http://mirrors.kernel.org/fedora-epel/ +http://mirrors.neusoft.edu.cn/epel/ http://mirrors.nl.eu.kernel.org/fedora-epel/ http://mirrors.ptd.net/epel/ http://mirrors.rit.edu/epel/ @@ -119,6 +127,7 @@ http://mirrors.syringanetworks.net/fedora-epel/ http://mirrors.tummy.com/pub/fedora.redhat.com/epel/ http://mirrors.ucr.ac.cr/epel/ +http://mirrors.ukfast.co.uk/sites/dl.fedoraproject.org/pub/epel/ http://mirrors.ustc.edu.cn/fedora/epel/ http://mirrors.xmission.com/fedora/epel/ http://nl.mirror.eurid.eu/epel/ diff -Nru apt-cacher-ng-0.7.7/conf/fedora_mirrors apt-cacher-ng-0.7.11/conf/fedora_mirrors --- apt-cacher-ng-0.7.7/conf/fedora_mirrors 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/conf/fedora_mirrors 2012-11-05 22:24:59.000000000 +0000 @@ -1,6 +1,8 @@ http://archive.linux.duke.edu/pub/fedora/linux/ http://be.mirror.eurid.eu/fedora/linux/ +http://chi-10g-1-mirror.fastsoft.net/pub/linux/fedora/linux/ http://dl.fedoraproject.org/pub/fedora/linux/ +http://falkor.skane.se/pub/mirrors/fedora/linux/ http://fedora-mirror01.rbc.ru/pub/fedora/linux/ http://fedora-mirror02.rbc.ru/pub/fedora/linux/ http://fedora.aau.at/linux/ @@ -11,6 +13,7 @@ http://fedora.inode.at/fedora/linux/ http://fedora.linuxman.biz/linux/ http://fedora.mirror.ac.za/linux/ +http://fedora.mirror.constant.com/linux/ http://fedora.mirror.garr.it/mirrors/fedora/linux/ http://fedora.mirror.nexicom.net/linux/ http://fedora.mirror.tn/fedora/linux/ @@ -45,20 +48,17 @@ http://ftp.iinet.net.au/pub/fedora/linux/ http://ftp.ines.lug.ro/fedora/linux/ http://ftp.isu.edu.tw/pub/Linux/Fedora/linux/ -http://ftp.itu.edu.tr/Mirror/Fedora/linux/ http://ftp.klid.dk/ftp/fedora/linux/ http://ftp.linux.cz/pub/linux/fedora/linux/ http://ftp.linux.ncsu.edu/pub/fedora/linux/ http://ftp.ncnu.edu.tw/Linux/Fedora/linux/ -http://ftp.ndlug.nd.edu/pub/fedora/linux/ http://ftp.nluug.nl/pub/os/Linux/distr/fedora/linux/ http://ftp.ntua.gr/pub/linux/fedora/linux/ http://ftp.ps.pl/pub/Linux/fedora-linux/ http://ftp.rhd.ru/pub/fedora/linux/ http://ftp.rhnet.is/pub/fedora/linux/ -http://ftp.roedu.net/mirrors/fedora.redhat.com/linux/ -http://ftp.sh.cvut.cz/MIRRORS/fedora/linux/ http://ftp.sjtu.edu.cn/fedora/linux/ +http://ftp.stu.edu.tw/Linux/Fedora/linux/ http://ftp.swin.edu.au/fedora/linux/ http://ftp.tsukuba.wide.ad.jp/Linux/fedora/linux/ http://ftp.tudelft.nl/download.fedora.redhat.com/linux/ @@ -71,7 +71,6 @@ http://ftp.usf.edu/pub/fedora/linux/ http://ftp.ussg.iu.edu/linux/fedora/linux/ http://ftp.wcss.pl/pub/linux/fedora/linux/ -http://ftp.yz.yamagata-u.ac.jp/pub/linux/fedora/linux/ http://gd.tuwien.ac.at/opsys/linux/fedora/linux/ http://kdeforge.unl.edu/mirrors/fedora/linux/ http://less.cogeco.net/fedora/linux/ @@ -81,15 +80,16 @@ http://mirror.anl.gov/pub/fedora/linux/ http://mirror.aptus.co.tz/pub/fedora/linux/ http://mirror.arlug.ro/pub/fedora/linux/ -http://mirror.as24220.net/pub/fedora/linux/ http://mirror.bytemark.co.uk/fedora/linux/ http://mirror.cc.vt.edu/pub/fedora/linux/ http://mirror.chpc.utah.edu/pub/fedora/linux/ +http://mirror.clarkson.edu/fedora/linux/ http://mirror.cogentco.com/pub/linux/fedora/linux/ http://mirror.cpsc.ucalgary.ca/mirror/fedora/linux/ http://mirror.csclub.uwaterloo.ca/fedora/linux/ http://mirror.datacenter.by/pub/fedoraproject.org/linux/ http://mirror.de.leaseweb.net/fedora/linux/ +http://mirror.fedora.delta-net.ge/pub/fedora/linux/ http://mirror.fraunhofer.de/download.fedora.redhat.com/fedora/linux/ http://mirror.hiwaay.net/pub/fedora/linux/ http://mirror.hmc.edu/fedora/linux/ @@ -97,16 +97,20 @@ http://mirror.i3d.net/pub/fedora/linux/ http://mirror.iprimus.com.au/fedora/linux/ http://mirror.its.dal.ca/pub/fedora/linux/ +http://mirror.its.sfu.ca/mirror/fedora/linux/ http://mirror.karneval.cz/pub/linux/fedora/linux/ +http://mirror.kiewel-online.ch/fedora/linux/ http://mirror.lib.ucdavis.edu/fedora/linux/ http://mirror.liberty.edu/pub/fedora/linux/ http://mirror.metrocast.net/fedora/linux/ http://mirror.nl.as6453.net/fedora/linux/ http://mirror.nl.leaseweb.net/fedora/linux/ +http://mirror.nus.edu.sg/fedora/linux/ http://mirror.optus.net/fedora/linux/ http://mirror.ox.ac.uk/sites/download.fedora.redhat.com/pub/fedora/linux/ http://mirror.pmf.kg.ac.rs/fedora/linux/ http://mirror.pnl.gov/fedora/linux/ +http://mirror.proserve.nl/fedora/linux/ http://mirror.seas.harvard.edu/fedora/linux/ http://mirror.squ.edu.om/fedora/linux/ http://mirror.switch.ch/ftp/mirror/fedora/linux/ @@ -128,8 +132,8 @@ http://mirrors.ircam.fr/pub/fedora/linux/ http://mirrors.issp.co.th/fedora/linux/ http://mirrors.linux.edu.lv/ftp.redhat.com/pub/fedora/linux/ +http://mirrors.n-ix.net/fedora/linux/ http://mirrors.ptd.net/fedora/linux/ -http://mirrors.reflected.net/fedora/linux/ http://mirrors.rit.edu/fedora/linux/ http://mirrors.tummy.com/pub/fedora.redhat.com/fedora/linux/ http://mirrors.usc.edu/pub/linux/distributions/fedora/linux/ @@ -144,6 +148,5 @@ http://ultra.linux.cz/MIRRORS/fedora.redhat.com/linux/ http://vesta.informatik.rwth-aachen.de/ftp/pub/Linux/fedora/linux/ http://www.gtlib.gatech.edu/pub/fedora.redhat/linux/ -http://www.las.ic.unicamp.br/pub/fedora/linux/ http://www.muug.mb.ca/pub/fedora/linux/ http://www.nic.funet.fi/pub/mirrors/fedora.redhat.com/pub/fedora/linux/ diff -Nru apt-cacher-ng-0.7.7/conf/report.html apt-cacher-ng-0.7.11/conf/report.html --- apt-cacher-ng-0.7.7/conf/report.html 2011-10-16 21:00:44.000000000 +0000 +++ apt-cacher-ng-0.7.11/conf/report.html 2012-12-16 18:56:41.000000000 +0000 @@ -100,7 +100,7 @@

- +

diff -Nru apt-cacher-ng-0.7.7/conf/sl_mirrors apt-cacher-ng-0.7.11/conf/sl_mirrors --- apt-cacher-ng-0.7.7/conf/sl_mirrors 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/conf/sl_mirrors 2012-11-01 23:07:31.000000000 +0000 @@ -6,6 +6,7 @@ http://ftp.halifax.rwth-aachen.de/scientificlinux/ http://ftp.heanet.ie/pub/scientific/ http://ftp.icm.edu.pl/pub/Linux/dist/scientific/ +http://ftp.itb.ac.id/pub/ScientificLinux/ http://ftp.jaist.ac.jp/pub/Linux/scientific/ http://ftp.kddilabs.jp/Linux/packages/scientificlinux/ http://ftp.lip6.fr/pub/linux/distributions/scientific/ @@ -14,7 +15,6 @@ http://ftp.scientificlinux.org/linux/scientific/ http://ftp.tsukuba.wide.ad.jp/Linux/scientific/ http://ftp.twaren.net/Linux/scientific/ -http://ftp.yz.yamagata-u.ac.jp/pub/linux/scientific/ http://ftp1.scientificlinux.org/linux/scientific/ http://ftp2.scientificlinux.org/linux/scientific/ http://linuxsoft.cern.ch/scientific/ @@ -28,10 +28,8 @@ http://mirror.optusnet.com.au/scientific/ http://mirror.squ.edu.om/scientificlinux/ http://mirror.switch.ch/ftp/mirror/scientificlinux/ -http://mirror.symnds.com/distributions/scientific/ http://mirrors.200p-sf.sonic.net/scientific/ http://nl.mirror.eurid.eu/scientific/ -http://rpm.scl.rs/linux/scientific/ http://scientificlinux.physik.uni-muenchen.de/mirror/scientific/ http://wftp.tu-chemnitz.de/pub/linux/scientific/ http://www.gtlib.gatech.edu/pub/scientific/ diff -Nru apt-cacher-ng-0.7.7/conf/ubuntu_mirrors apt-cacher-ng-0.7.11/conf/ubuntu_mirrors --- apt-cacher-ng-0.7.7/conf/ubuntu_mirrors 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/conf/ubuntu_mirrors 2012-11-05 22:24:59.000000000 +0000 @@ -16,9 +16,9 @@ http://archive.mmu.edu.my/ubuntu/ http://archive.ubuntu.com.ba/ubuntu/ http://archive.ubuntu.com/ubuntu/ +http://archive.ubuntu.csg.uzh.ch/ubuntu/ http://archive.ubuntu.mirror.dkm.cz/ http://archive.ubuntu.nautile.nc/ubuntu/ -http://archive.ubuntu.uasw.edu/ http://archive.ubuntumirror.dei.uc.pt/ubuntu/ http://artfiles.org/ubuntu.com/ http://as.archive.ubuntu.com/ubuntu/ @@ -26,9 +26,11 @@ http://au.archive.ubuntu.com/ubuntu/ http://aw.archive.ubuntu.com/ubuntu/ http://ax.archive.ubuntu.com/ubuntu/ +http://az-1.hpcloud.mirror.websitedevops.com/ubuntu/ http://az.archive.ubuntu.com/ubuntu/ http://ba.archive.ubuntu.com/ubuntu/ http://bb.archive.ubuntu.com/ubuntu/ +http://bd.archive.ubuntu.com/ubuntu/ http://be.archive.ubuntu.com/ubuntu/ http://bf.archive.ubuntu.com/ubuntu/ http://bg.archive.ubuntu.com/ubuntu/ @@ -92,7 +94,6 @@ http://eg.archive.ubuntu.com/ubuntu/ http://eh.archive.ubuntu.com/ubuntu/ http://er.archive.ubuntu.com/ubuntu/ -http://es.archive.ubuntu.com/ubuntu/ http://espelhos.edugraf.ufsc.br/ubuntu/ http://et.archive.ubuntu.com/ubuntu/ http://eu.archive.ubuntu.com/ubuntu/ @@ -110,12 +111,14 @@ http://ftp.astral.ro/mirrors/ubuntu.com/ubuntu/ http://ftp.availo.se/ubuntu/ http://ftp.belnet.be/mirror/ubuntu.com/ubuntu/ +http://ftp.byfly.by/ubuntu/ http://ftp.caliu.cat/pub/distribucions/ubuntu/archive/ http://ftp.cc.uoc.gr/mirrors/linux/ubuntu/packages/ http://ftp.ccc.uba.ar/pub/linux/ubuntu/ http://ftp.citylink.co.nz/ubuntu/ http://ftp.crihan.fr/ubuntu/ http://ftp.cs.pu.edu.tw/Linux/Ubuntu/ubuntu/ +http://ftp.cuhk.edu.hk/pub/Linux/ubuntu/ http://ftp.cvut.cz/ubuntu/ http://ftp.dat.etsit.upm.es/ubuntu/ http://ftp.daum.net/ubuntu/ @@ -149,10 +152,8 @@ http://ftp.portlane.com/ubuntu/ http://ftp.riken.jp/Linux/ubuntu/ http://ftp.rnl.ist.utl.pt/pub/ubuntu/archive/ -http://ftp.roedu.net/mirrors/ubuntulinux.org/ubuntu/ http://ftp.rrzn.uni-hannover.de/pub/mirror/linux/ubuntu/ http://ftp.rz.tu-bs.de/pub/mirror/ubuntu-packages/ -http://ftp.sh.cvut.cz/MIRRORS/ubuntu/ http://ftp.sjtu.edu.cn/ubuntu/ http://ftp.snt.utwente.nl/pub/os/linux/ubuntu/ http://ftp.stw-bonn.de/ubuntu/ @@ -170,7 +171,7 @@ http://ftp.tudelft.nl/archive.ubuntu.com/ http://ftp.twaren.net/Linux/Ubuntu/ubuntu/ http://ftp.u-picardie.fr/mirror/ubuntu/ubuntu/ -http://ftp.uni-bayreuth.de/linux/ubuntu/ubuntu/ +http://ftp.udc.es/ubuntu/ http://ftp.uni-erlangen.de/mirrors/ubuntu/ http://ftp.uni-kassel.de/ubuntu/ubuntu/ http://ftp.uni-kl.de/pub/linux/ubuntu/ @@ -181,8 +182,8 @@ http://ftp.ussg.iu.edu/linux/ubuntu/ http://ftp.utexas.edu/ubuntu/ http://ftp.vectranet.pl/ubuntu/ +http://ftp.wa.co.za/pub/ubuntu/ubuntu/ http://ftp.wcss.pl/ubuntu/ -http://ftp.yz.yamagata-u.ac.jp/pub/linux/ubuntu/archives/ http://ftp5.gwdg.de/pub/linux/debian/ubuntu/ http://ga.archive.ubuntu.com/ubuntu/ http://gaosu.rave.org/ubuntu/ @@ -208,6 +209,7 @@ http://gu.archive.ubuntu.com/ubuntu/ http://gw.archive.ubuntu.com/ubuntu/ http://gy.archive.ubuntu.com/ubuntu/ +http://hive.ist.unomaha.edu/ubuntu-archive/ http://hk.archive.ubuntu.com/ubuntu/ http://hm.archive.ubuntu.com/ubuntu/ http://hn.archive.ubuntu.com/ubuntu/ @@ -222,6 +224,7 @@ http://io.archive.ubuntu.com/ubuntu/ http://iq.archive.ubuntu.com/ubuntu/ http://ir.archive.ubuntu.com/ubuntu/ +http://is.archive.ubuntu.com/ubuntu/ http://it.archive.ubuntu.com/ubuntu/ http://je.archive.ubuntu.com/ubuntu/ http://jm.archive.ubuntu.com/ubuntu/ @@ -230,7 +233,6 @@ http://kambing.ui.ac.id/ubuntu/ http://kartolo.sby.datautama.net.id/ubuntu/ http://ke.archive.ubuntu.com/ubuntu/ -http://kebo.vlsm.org/ubuntu/ http://kg.archive.ubuntu.com/ubuntu/ http://kh.archive.ubuntu.com/ubuntu/ http://ki.archive.ubuntu.com/ubuntu/ @@ -273,11 +275,10 @@ http://mirror.as24220.net/pub/ubuntu/archive/ http://mirror.as29550.net/archive.ubuntu.com/ http://mirror.bauhuette.fh-aachen.de/ubuntu/ -http://mirror.bjtu.edu.cn/ubuntu/ http://mirror.blizoo.mk/ubuntu/ http://mirror.bytemark.co.uk/ubuntu/ -http://mirror.calvin.edu/ubuntu/ http://mirror.cc.columbia.edu/pub/linux/ubuntu/archive/ +http://mirror.clarkson.edu/ubuntu/ http://mirror.clibre.uqam.ca/ubuntu/ http://mirror.cogentco.com/pub/linux/ubuntu/ http://mirror.corbina.net/ubuntu/ @@ -289,23 +290,25 @@ http://mirror.datacenter.by/ubuntu/ http://mirror.dattobackup.com/ubuntu/ http://mirror.de.leaseweb.net/ubuntu/ +http://mirror.dhakacom.com/ubuntu/ http://mirror.edatel.net.co/ubuntu/ http://mirror.fcaglp.unlp.edu.ar/ubuntu/ http://mirror.globo.com/ubuntu/archive/ http://mirror.greennet.gl/ubuntu/ http://mirror.hmc.edu/ubuntu/ http://mirror.i3d.net/pub/ubuntu/ +http://mirror.ihug.co.nz/ubuntu/ http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/ http://mirror.internode.on.net/pub/ubuntu/ubuntu/ http://mirror.isoc.org.il/pub/ubuntu/ http://mirror.its.dal.ca/ubuntu/ http://mirror.its.sfu.ca/mirror/ubuntu/ +http://mirror.jmu.edu/pub/ubuntu/ http://mirror.kku.ac.th/ubuntu/ http://mirror.krystal.co.uk/ubuntu/ http://mirror.lcsee.wvu.edu/ubuntu/ http://mirror.learn.ac.lk/ubuntu/ http://mirror.lihnidos.org/ubuntu/ubuntu/ -http://mirror.linux.org.au/ubuntu/ http://mirror.linux.org.mt/ubuntu/ http://mirror.lstn.net/ubuntu/ http://mirror.lupaworld.com/ubuntu/ @@ -318,8 +321,9 @@ http://mirror.netlinux.cl/ubuntu/ http://mirror.neu.edu.cn/ubuntu/ http://mirror.nl.leaseweb.net/ubuntu/ +http://mirror.nus.edu.sg/ubuntu/ http://mirror.optus.net/ubuntu/ -http://mirror.oscc.org.my/ubuntu/ +http://mirror.overthewire.com.au/ubuntu/ http://mirror.ovh.net/ubuntu/ http://mirror.ox.ac.uk/sites/archive.ubuntu.com/ubuntu/ http://mirror.peer1.net/ubuntu/ @@ -327,9 +331,9 @@ http://mirror.pnl.gov/ubuntu/ http://mirror.pregi.net/ubuntu/ http://mirror.qdu.edu.cn/ubuntu/ -http://mirror.quickvz.com/ubuntu/ http://mirror.rol.ru/ubuntu/ http://mirror.serverloft.eu/ubuntu/ubuntu/ +http://mirror.siamdata.co.th/ubuntu/ http://mirror.soften.ktu.lt/ubuntu/ http://mirror.sov.uk.goscomb.net/ubuntu/ http://mirror.squ.edu.om/ubuntuarchive/ @@ -354,6 +358,7 @@ http://mirror2.corbina.ru/ubuntu/ http://mirrors.163.com/ubuntu/ http://mirrors.accretive-networks.net/ubuntu/ +http://mirrors.arpnetworks.com/Ubuntu/ http://mirrors.bloomu.edu/ubuntu/ http://mirrors.cat.pdx.edu/ubuntu/ http://mirrors.ccs.neu.edu/ubuntu/ @@ -362,32 +367,32 @@ http://mirrors.cs.wmich.edu/ubuntu/ http://mirrors.dotsrc.org/ubuntu/ http://mirrors.easynews.com/linux/ubuntu/ -http://mirrors.ecvps.com/ubuntu/ http://mirrors.einstein.yu.edu/ubuntu/archive/ http://mirrors.fe.up.pt/ubuntu/ http://mirrors.gigenet.com/ubuntuarchive/ http://mirrors.ircam.fr/pub/ubuntu/archive/ +http://mirrors.ispros.com.bd/ubuntu/ http://mirrors.melbourne.co.uk/ubuntu/ http://mirrors.mit.edu/ubuntu/ +http://mirrors.neusoft.edu.cn/ubuntu/ http://mirrors.nfsi.pt/ubuntu/ http://mirrors.nic.funet.fi/ubuntu/ http://mirrors.nl.eu.kernel.org/ubuntu/ http://mirrors.psu.ac.th/pub/ubuntu/ http://mirrors.rit.edu/ubuntu/ http://mirrors.se.eu.kernel.org/ubuntu/ -http://mirrors.serverhost.ro/ubuntu/archive/ http://mirrors.sohu.com/ubuntu/ +http://mirrors.syringanetworks.net/ubuntu-archive/ http://mirrors.tecnoera.com/ubuntu/ http://mirrors.telianet.dk/ubuntu/ +http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ http://mirrors.uaip.org/ubuntu/ -http://mirrors.ucr.ac.cr/ubuntu/ http://mirrors.us.kernel.org/ubuntu/ http://mirrors.ustc.edu.cn/ubuntu/ http://mirrors.xmission.com/ubuntu/ http://mk.archive.ubuntu.com/ubuntu/ http://ml.archive.ubuntu.com/ubuntu/ http://mm.archive.ubuntu.com/ubuntu/ -http://mn.archive.ubuntu.com/ubuntu/ http://mo.archive.ubuntu.com/ubuntu/ http://mosel.estg.ipleiria.pt/mirror/distros/ubuntu/archive/ http://mp.archive.ubuntu.com/ubuntu/ @@ -417,7 +422,6 @@ http://om.archive.ubuntu.com/ubuntu/ http://osmirror.rug.nl/ubuntu/ http://pa.archive.ubuntu.com/ubuntu/ -http://pandawa.ipb.ac.id/ubuntu/ http://pe.archive.ubuntu.com/ubuntu/ http://pf.archive.ubuntu.com/ubuntu/ http://pg.archive.ubuntu.com/ubuntu/ @@ -433,15 +437,13 @@ http://py.archive.ubuntu.com/ubuntu/ http://qa.archive.ubuntu.com/ubuntu/ http://re.archive.ubuntu.com/ubuntu/ +http://releases.ubuntu.spd.co.il/ http://ro.archive.ubuntu.com/ubuntu/ -http://rpm.scl.rs/linux/ubuntu/archive/ http://rs.archive.ubuntu.com/ubuntu/ http://ru.archive.ubuntu.com/ubuntu/ -http://russell.cs.bilgi.edu.tr/ubuntu/ http://rw.archive.ubuntu.com/ubuntu/ http://sa.archive.ubuntu.com/ubuntu/ http://samaritan.ucmerced.edu/ubuntu/ -http://san.csc.calpoly.edu/ubuntu/ubuntu/ http://sb.archive.ubuntu.com/ubuntu/ http://sc.archive.ubuntu.com/ubuntu/ http://sd.archive.ubuntu.com/ubuntu/ @@ -458,6 +460,7 @@ http://sm.archive.ubuntu.com/ubuntu/ http://sn.archive.ubuntu.com/ubuntu/ http://so.archive.ubuntu.com/ubuntu/ +http://softlibre.unizar.es/ubuntu/archive/ http://speglar.simnet.is/ubuntu/ http://sr.archive.ubuntu.com/ubuntu/ http://ss.archive.ubuntu.com/ubuntu/ @@ -492,20 +495,19 @@ http://tz.archive.ubuntu.com/ubuntu/ http://ua.archive.ubuntu.com/ubuntu/ http://ubuntu-arch.linux.edu.lv/ubuntu/ -http://ubuntu-archive.adsolux.com/ubuntu/ http://ubuntu-archive.locaweb.com.br/ubuntu/ +http://ubuntu-archive.mirror.nucleus.be/ http://ubuntu-archive.mirrors.proxad.net/ubuntu/ +http://ubuntu-archive.polytechnic.edu.na/ubuntu/ http://ubuntu-archive.sit.kmutt.ac.th/ http://ubuntu-archives.mirror.nexicom.net/ http://ubuntu-ashisuto.ubuntulinux.jp/ubuntu/ http://ubuntu-mirror.telesys.org.ua/ubuntu/ http://ubuntu.arcticnetwork.ca/ http://ubuntu.c3sl.ufpr.br/ubuntu/ -http://ubuntu.cic.userena.cl/ubuntu/ http://ubuntu.cica.es/ubuntu/ http://ubuntu.cn99.com/ubuntu/ http://ubuntu.cnssuestc.org/ubuntu/ -http://ubuntu.cs.nctu.edu.tw/ubuntu/ http://ubuntu.cs.utah.edu/ubuntu/ http://ubuntu.ctu.edu.vn/archive/ http://ubuntu.cybercomhosting.com/ubuntu/ @@ -526,12 +528,15 @@ http://ubuntu.laps.ufpa.br/ubuntu/ http://ubuntu.linux-bg.org/ubuntu/ http://ubuntu.load.lv/ubuntu/ +http://ubuntu.mirror.ac.ke/ubuntu/ http://ubuntu.mirror.ac.za/ubuntu-archive/ http://ubuntu.mirror.atratoip.net/ubuntu/ http://ubuntu.mirror.cambrium.nl/ubuntu/ +http://ubuntu.mirror.constant.com/ http://ubuntu.mirror.frontiernet.net/ubuntu/ http://ubuntu.mirror.garr.it/mirrors/ubuntu-archive/ http://ubuntu.mirror.iweb.ca/ +http://ubuntu.mirror.neology.co.za/ubuntu/ http://ubuntu.mirror.netelligent.ca/ubuntu/ http://ubuntu.mirror.pop-sc.rnp.br/ubuntu/ http://ubuntu.mirror.rafal.ca/ubuntu/ @@ -539,6 +544,7 @@ http://ubuntu.mirror.su.se/ubuntu/ http://ubuntu.mirror.tn/ http://ubuntu.mirror.tudos.de/ubuntu/ +http://ubuntu.mirror.uber.com.au/archive/ http://ubuntu.mirror.vu.lt/ubuntu/ http://ubuntu.mirrors.crysys.hu/ http://ubuntu.mirrors.isu.net.sa/ubuntu/ @@ -549,6 +555,7 @@ http://ubuntu.mirrors.uk2.net/ubuntu/ http://ubuntu.mithril-linux.org/archives/ http://ubuntu.ntc.net.np/ubuntu/ +http://ubuntu.org.ua/ubuntu/ http://ubuntu.oss.eznetsols.org/ubuntu/ http://ubuntu.osuosl.org/ubuntu/ http://ubuntu.otenet.gr/ @@ -561,11 +568,11 @@ http://ubuntu.sastudio.jp/ubuntu/ http://ubuntu.secs.oakland.edu/ http://ubuntu.securedservers.com/ -http://ubuntu.sh.cvut.cz/ http://ubuntu.skarta.net/ubuntu/ http://ubuntu.snet.uz/ubuntu/ http://ubuntu.srt.cn/ubuntu/ http://ubuntu.sth.sze.hu/ubuntu/ +http://ubuntu.stu.edu.tw/ubuntu/ http://ubuntu.task.gda.pl/ubuntu/ http://ubuntu.tiscali.nl/ http://ubuntu.trumpetti.atm.tut.fi/ubuntu/ @@ -573,6 +580,7 @@ http://ubuntu.tuxuri.com/ubuntu/ http://ubuntu.uach.mx/ http://ubuntu.uc3m.es/ubuntu/ +http://ubuntu.ucr.ac.cr/ubuntu/ http://ubuntu.uestc.edu.cn/ubuntu/ http://ubuntu.ufba.br/ubuntu/ http://ubuntu.uib.no/archive/ @@ -586,8 +594,8 @@ http://ubuntu.virginmedia.com/archive/ http://ubuntu.wallawalla.edu/ubuntu/ http://ubuntu.wikimedia.org/ubuntu/ +http://ubuntu.xfree.com.ar/ubuntu/ http://ubuntuarchive.hnsdc.com/ubuntu/ -http://ubuntuarchive.xfree.com.ar/ubuntu/ http://ubuntutym.u-toyama.ac.jp/ubuntu/ http://ucho.ignum.cz/ubuntu/ http://ucmirror.canterbury.ac.nz/ubuntu/ diff -Nru apt-cacher-ng-0.7.7/contrib/apt-cacher-ng-mirror apt-cacher-ng-0.7.11/contrib/apt-cacher-ng-mirror --- apt-cacher-ng-0.7.7/contrib/apt-cacher-ng-mirror 1970-01-01 00:00:00.000000000 +0000 +++ apt-cacher-ng-0.7.11/contrib/apt-cacher-ng-mirror 2012-11-04 18:27:33.000000000 +0000 @@ -0,0 +1,22 @@ +#!/bin/sh +# Sample script for execution as part of a cron job +# Triggers precaching of remote contents, see manual for details + +# optional, specify hostname to appear in error messages and links +# HOSTNAME=externallyVisibleHostName +# export HOSTNAME + +# optional, specify IP address to connect to +# ACNGIP=10.0.1.3 +# export ACNGIP + +ACNGREQ='?doDownload=dd&asNeeded=an&useDebDelta=ud&doMirror=Start+Mirroring' +export ACNGREQ + +if perl -e 'use MIME::Base64' 2>/dev/null \ + && test -r /usr/lib/apt-cacher-ng/expire-caller.pl ; +then + perl /usr/lib/apt-cacher-ng/expire-caller.pl || echo Please check error messages in one of: $(ls -t /var/log/apt-cacher-ng/maint*.log.html | head -n5) +else + echo $(date -R)\|/etc/cron.daily/apt-cacher-ng\| Error starting expiration helper. Install the perl package, check apt-cacher-ng installation. >> /var/log/apt-cacher-ng/apt-cacher.err +fi diff -Nru apt-cacher-ng-0.7.7/dbgen/sig-debian apt-cacher-ng-0.7.11/dbgen/sig-debian --- apt-cacher-ng-0.7.7/dbgen/sig-debian 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/dbgen/sig-debian 2012-11-05 22:24:59.000000000 +0000 @@ -1 +1 @@ -b927c3f7bc7e5bd62f1723e4d110147b dbgen/tmp/dsnap +e2a1776197eaa14952932ccc134cb666 dbgen/tmp/dsnap diff -Nru apt-cacher-ng-0.7.7/dbgen/sig-fsnap apt-cacher-ng-0.7.11/dbgen/sig-fsnap --- apt-cacher-ng-0.7.7/dbgen/sig-fsnap 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/dbgen/sig-fsnap 2012-11-05 22:24:59.000000000 +0000 @@ -1 +1 @@ -267eca24d787402c11d2dbce5188ac90 dbgen/tmp/fsnap +4a8480a6ef17b5f71cf5a48d97d529b7 dbgen/tmp/fsnap diff -Nru apt-cacher-ng-0.7.7/dbgen/sig-slsnap apt-cacher-ng-0.7.11/dbgen/sig-slsnap --- apt-cacher-ng-0.7.7/dbgen/sig-slsnap 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/dbgen/sig-slsnap 2012-11-01 23:07:31.000000000 +0000 @@ -1 +1 @@ -7333d03841acdf5dfb01be93218062b0 dbgen/tmp/slsnap +f8024c56bd922237281492eac49bd371 dbgen/tmp/slsnap diff -Nru apt-cacher-ng-0.7.7/dbgen/sig-ubuntu apt-cacher-ng-0.7.11/dbgen/sig-ubuntu --- apt-cacher-ng-0.7.7/dbgen/sig-ubuntu 2012-08-02 18:20:58.000000000 +0000 +++ apt-cacher-ng-0.7.11/dbgen/sig-ubuntu 2012-11-05 22:24:59.000000000 +0000 @@ -1 +1 @@ -b6dc7001a27e2632f82854b43d51c8f3 dbgen/tmp/usnap +744c5ada6ca841cd793ec41e2a1ff299 dbgen/tmp/usnap diff -Nru apt-cacher-ng-0.7.7/dbgen/ubuntuscan.sh apt-cacher-ng-0.7.11/dbgen/ubuntuscan.sh --- apt-cacher-ng-0.7.7/dbgen/ubuntuscan.sh 2011-10-09 19:23:47.000000000 +0000 +++ apt-cacher-ng-0.7.11/dbgen/ubuntuscan.sh 2012-11-01 23:07:31.000000000 +0000 @@ -13,7 +13,7 @@ tempdir=${3:-$(mktemp -d)} sfx=${4:-/dists} # Debian is easy, Ubuntu is bad... no symbolic name. Try to match -security, or one of the latest release names -testkey=${5:-'updates\|stable\|unstable\|security\|backports\|feisty\|gutsy\|hardy\|intrepid\|jaunty\|karmic\|lucid\|maverick\|depper\|natty\|oneiric'} +testkey=${5:-'updates\|stable\|unstable\|security\|backports\|feisty\|gutsy\|hardy\|intrepid\|jaunty\|karmic\|lucid\|maverick\|depper\|natty\|oneiric\|dapper\|edgy\|precise\|quantal'} mkdir -p $tempdir rm -f $tempdir/url.* $tempdir/log.* diff -Nru apt-cacher-ng-0.7.7/debian/apt-cacher-ng.cron.daily apt-cacher-ng-0.7.11/debian/apt-cacher-ng.cron.daily --- apt-cacher-ng-0.7.7/debian/apt-cacher-ng.cron.daily 2012-08-05 14:27:38.000000000 +0000 +++ apt-cacher-ng-0.7.11/debian/apt-cacher-ng.cron.daily 2012-11-04 18:20:50.000000000 +0000 @@ -13,7 +13,7 @@ if perl -e 'use MIME::Base64' 2>/dev/null \ && test -r /usr/lib/apt-cacher-ng/expire-caller.pl ; then - perl /usr/lib/apt-cacher-ng/expire-caller.pl || echo Check error messages in /var/log/apt-cacher-ng/maint_*.log + perl /usr/lib/apt-cacher-ng/expire-caller.pl || echo Please check error messages in one of: $(ls -t /var/log/apt-cacher-ng/maint*.log.html | head -n5) else echo $(date -R)\|/etc/cron.daily/apt-cacher-ng\| Error starting expiration helper. Install the perl package, check apt-cacher-ng installation. >> /var/log/apt-cacher-ng/apt-cacher.err fi diff -Nru apt-cacher-ng-0.7.7/debian/changelog apt-cacher-ng-0.7.11/debian/changelog --- apt-cacher-ng-0.7.7/debian/changelog 2012-10-21 03:40:12.000000000 +0000 +++ apt-cacher-ng-0.7.11/debian/changelog 2013-05-16 18:24:20.000000000 +0000 @@ -1,18 +1,49 @@ -apt-cacher-ng (0.7.7-1ubuntu1~ubuntu12.04.1) precise-backports; urgency=low +apt-cacher-ng (0.7.11-1~ubuntu12.04.1) precise-backports; urgency=low - * No-change backport to precise (LP: #1068917) + * Backport to precise (LP: #1172930) + * Remove logrotate conflict + * debian/apt-cacher-ng.logrotate: + Comment out the "su" lines as this requires logrotate 3.8.1 which isn't in + Precise. - -- Micah Gersten Sat, 20 Oct 2012 22:40:12 -0500 + -- Adam Stokes Thu, 16 May 2013 20:23:41 +0200 -apt-cacher-ng (0.7.7-1ubuntu1) quantal; urgency=low +apt-cacher-ng (0.7.11-1) unstable; urgency=low - * Merge from Debian. Remaining changes: - - debian/apt-cacher-ng.logrotate: - Comment out the "su" lines as this requires logrotate 3.8.1 which isn't - in Ubuntu yet. - - drop conflict on logrotate < 3.8.1 + * New upstream release + + fixes a segfaulting bug within the new code (closes: #694620) + + spelling correction in the web interface (threat/treat, closes: #695120) + + -- Eduard Bloch Wed, 19 Dec 2012 20:14:46 +0100 + +apt-cacher-ng (0.7.10-1) unstable; urgency=medium + + * New upstream release + + -- Eduard Bloch Mon, 05 Nov 2012 23:27:29 +0100 + +apt-cacher-ng (0.7.9-1) unstable; urgency=low + + * New upstream release + + fixes regular downloads in ForceManaged mode, closes: #692373) + + -- Eduard Bloch Mon, 05 Nov 2012 23:17:06 +0100 + +apt-cacher-ng (0.7.8-1) unstable; urgency=low + + * New upstream release + + improved redirection handling (closes: #683803) + + workarounds for stuck download descriptors (now probably really + closes:#677983) + * Cron jobs displays max. 5 recent operation logs (LP: #1074474) + + -- Eduard Bloch Sun, 04 Nov 2012 20:36:32 +0100 + +apt-cacher-ng (0.7.7-2) unstable; urgency=low + + * Unstable upload (no changes, hope that it closes: #677983) - -- Jeremy Bicha Sun, 05 Aug 2012 09:54:58 -0400 + -- Eduard Bloch Sat, 08 Sep 2012 18:17:22 +0200 apt-cacher-ng (0.7.7-1) experimental; urgency=low diff -Nru apt-cacher-ng-0.7.7/debian/control apt-cacher-ng-0.7.11/debian/control --- apt-cacher-ng-0.7.7/debian/control 2012-08-05 14:27:38.000000000 +0000 +++ apt-cacher-ng-0.7.11/debian/control 2013-05-08 16:32:40.000000000 +0000 @@ -1,8 +1,7 @@ Source: apt-cacher-ng Section: net Priority: optional -Maintainer: Ubuntu Developers -XSBC-Original-Maintainer: Eduard Bloch +Maintainer: Eduard Bloch Build-Depends: debhelper (>= 7.2.3), cmake (>= 2.6.2), libbz2-dev, zlib1g-dev, liblzma-dev, libfuse-dev [!hurd-i386], pkg-config, libwrap0-dev Build-Conflicts: binutils-gold Standards-Version: 3.9.3 @@ -13,7 +12,6 @@ Depends: ${shlibs:Depends}, ${misc:Depends}, adduser Pre-Depends: dpkg (>= 1.15.6) Recommends: perl (>> 5.8), ed -#Conflicts: logrotate (<< 3.8.0) Suggests: doc-base, libfuse2 (>= 2.5) Description: caching proxy server for software repositories Apt-Cacher NG is a caching proxy for downloading packages from Debian-style diff -Nru apt-cacher-ng-0.7.7/doc/apt-cacher-ng.pdf apt-cacher-ng-0.7.11/doc/apt-cacher-ng.pdf --- apt-cacher-ng-0.7.7/doc/apt-cacher-ng.pdf 2012-08-04 18:39:00.000000000 +0000 +++ apt-cacher-ng-0.7.11/doc/apt-cacher-ng.pdf 2012-12-19 19:18:06.000000000 +0000 @@ -4,7 +4,7 @@ 1 0 obj << /Title (Apt-Cacher-NG User Manual) -/Producer (Halibut, anonymous build \(Jun 22 2012 12:14:14\)) +/Producer (Halibut, anonymous build \(Nov 23 2012 02:02:20\)) >> endobj 2 0 obj diff -Nru apt-cacher-ng-0.7.7/fs/httpfs.cc apt-cacher-ng-0.7.11/fs/httpfs.cc --- apt-cacher-ng-0.7.7/fs/httpfs.cc 2012-08-04 17:50:47.000000000 +0000 +++ apt-cacher-ng-0.7.11/fs/httpfs.cc 2012-11-04 16:42:27.000000000 +0000 @@ -64,9 +64,6 @@ #define POOLMAXSIZE 20 // max size #define POOLMAXAGE 50 // seconds - -//const time_t cleaner::never=ULONG_MAX; - // some globals, set only once static struct stat statTempl; static struct statfs stfsTemp; diff -Nru apt-cacher-ng-0.7.7/include/acfg.h apt-cacher-ng-0.7.11/include/acfg.h --- apt-cacher-ng-0.7.7/include/acfg.h 2012-07-23 13:38:14.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/acfg.h 2012-11-04 16:42:27.000000000 +0000 @@ -10,6 +10,7 @@ static const int RESERVED_DEFVAL = -4223; +#define ACFG_REDIRMAX_DEFAULT 5 struct ltstring { @@ -31,7 +32,7 @@ extern int debug, numcores, offlinemode, foreground, verbose, stupidfs, forcemanaged, keepnver, verboselog, extreshhold, exfailabort, tpstandbymax, tpthreadmax, dnscachetime, dlbufsize, usewrap, exporigin, logxff, oldupdate, recompbz2, nettimeout, updinterval, forwardsoap, dirperms, fileperms, -maxtempdelay, redirmax, vrangeops; +maxtempdelay, redirmax, vrangeops, stucksecs; // processed config settings extern tHttpUrl proxy_info; diff -Nru apt-cacher-ng-0.7.7/include/cacheman.h apt-cacher-ng-0.7.11/include/cacheman.h --- apt-cacher-ng-0.7.7/include/cacheman.h 2012-05-24 11:41:45.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/cacheman.h 2012-11-06 22:26:24.000000000 +0000 @@ -23,8 +23,6 @@ #define SZABSPATH(x) (CACHE_BASE+(x)).c_str() #define SABSPATH(x) (CACHE_BASE+(x)) -#define REDIRMAX acfg::redirmax - #define FAKEDATEMARK "Sat, 26 Apr 1986 01:23:39 GMT+3" // caching all relevant file identity data and helper flags in such entries @@ -115,7 +113,7 @@ EIDX_ARCHLXDB, EIDX_CYGSETUP, EIDX_SUSEREPO, - EIDX_SUSEOTHER, + EIDX_XMLRPMLIST, EIDX_RFC822WITHLISTS, EIDX_TRANSIDX }; diff -Nru apt-cacher-ng-0.7.7/include/cleaner.h apt-cacher-ng-0.7.11/include/cleaner.h --- apt-cacher-ng-0.7.7/include/cleaner.h 2012-08-04 18:36:33.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/cleaner.h 2012-11-04 16:42:27.000000000 +0000 @@ -19,12 +19,9 @@ virtual ~cleaner(); void RunLatestAt(time_t when); - void WorkLoop(); void Stop(); - inline static const time_t never() { return MYSTD:: numeric_limits::max(); }; - private: time_t m_nextRun; pthread_t m_thr; diff -Nru apt-cacher-ng-0.7.7/include/config.h apt-cacher-ng-0.7.11/include/config.h --- apt-cacher-ng-0.7.7/include/config.h 2012-08-04 18:39:00.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/config.h 2012-12-19 19:18:06.000000000 +0000 @@ -4,7 +4,7 @@ #include "acsyscap.h" -#define ACVERSION "0.7.7" +#define ACVERSION "0.7.11" #define __STDC_FORMAT_MACROS #include diff -Nru apt-cacher-ng-0.7.7/include/conn.h apt-cacher-ng-0.7.11/include/conn.h --- apt-cacher-ng-0.7.7/include/conn.h 2012-05-02 21:57:44.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/conn.h 2012-11-01 22:35:03.000000000 +0000 @@ -71,8 +71,6 @@ #ifdef DEBUG UINT m_nProcessedJobs; #endif - - job * AddNewJob(header *tempHeader); }; #endif diff -Nru apt-cacher-ng-0.7.7/include/dlcon.h apt-cacher-ng-0.7.11/include/dlcon.h --- apt-cacher-ng-0.7.7/include/dlcon.h 2012-02-17 19:30:57.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/dlcon.h 2012-11-04 09:35:00.000000000 +0000 @@ -65,6 +65,18 @@ UINT ExchangeData(mstring &sErrorMsg, tTcpHandlePtr &con, tDljQueue &qActive); + // Disable pipelining for the next # requests. Actually used as crude workaround for the + // concept limitation (because of automata over a couple of function) and its + // impact on download performance. + // The use case: stupid web servers that redirect all requests do that step-by-step, i.e. + // they get a bunch of requests but return only the first response and then flush the buffer + // so we process this response and wish to switch to the new target location (dropping + // the current connection because we don't keep it somehow to background, this is the only + // download agent we have). This manner perverts the whole principle and causes permanent + // disconnects/reconnects. In this case, it's beneficial to disable pipelining and send + // our requests one-by-one. This is done for a while (i.e. the valueof(m_nDisablePling)/2 ) + // times before the operation mode returns to normal. + int m_nDisablePling; }; #endif diff -Nru apt-cacher-ng-0.7.7/include/fileitem.h apt-cacher-ng-0.7.11/include/fileitem.h --- apt-cacher-ng-0.7.7/include/fileitem.h 2012-06-07 15:24:37.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/fileitem.h 2012-11-04 17:16:30.000000000 +0000 @@ -68,7 +68,6 @@ uint64_t m_nIncommingCount; off_t m_nSizeSeen; - mstring m_sPathAbs, m_sPathRel; bool m_bCheckFreshness; @@ -91,6 +90,10 @@ time_t m_nTimeExpireAt; //! when to consider this item obsolete static time_t m_nEarliestExpiration; //! when to consider a closer look for expired items + mstring m_sPathAbs, m_sPathRel; + + time_t m_nTimeDlDone; + public: virtual void Unreg() {}; // to decrease dependent user count when used in global contents @@ -108,6 +111,7 @@ { public: inline fileitem_with_storage(cmstring &s) : fileitem_base(s) {}; + virtual ~fileitem_with_storage(); // send helper like wrapper for sendfile. Just declare virtual here to make it better customizable later. virtual ssize_t SendData(int confd, int filefd, off_t &nSendPos, size_t nMax2SendNow); virtual SuccessWithTransErrorFlag DownloadStartedStoreHeader(const header & h, const char *pNextData, @@ -115,7 +119,8 @@ virtual bool StoreFileData(const char *data, unsigned int size); }; -class fileitem; +typedef MYSTD::pair tFileRefEntry; +typedef MYSTD::multimap tFiGlobMap; //! This is a fileitem class which is aware of other items and download sources, //! includes global download state management @@ -124,27 +129,24 @@ public: virtual void Unreg(); // decrease dependent user count - //! Call Unreg directly, or return immediately but call it few seconds later - static void UnregOneASAP(tFileItemPtr); - // related to GetRegisteredFileItem but used for registration of custom file item // implementations created elsewhere (which still need to obey regular work flow) static bool RegisterFileItem(SHARED_PTR spCustomFileItem); - // public constructor wrapper, get a unique object from the map - static tFileItemPtr GetRegisteredFileItem(mstring sPath); // also opens the cached file and increases reference count + // public constructor wrapper, get a unique object from the map or a new one + static tFileItemPtr GetRegisteredFileItem(cmstring &sPath, bool bConsiderAltStore); //! @return: true iff there is still something in the pool for later cleaning static time_t DoDelayedUnregAndCheck(); -protected: - inline fileitem(cmstring s) : fileitem_with_storage(s) {}; + static void dump_status(); +protected: + fileitem(cmstring &s); + tFiGlobMap::iterator m_globRef; }; #endif -extern lockable mapLck; - #endif diff -Nru apt-cacher-ng-0.7.7/include/header.h apt-cacher-ng-0.7.11/include/header.h --- apt-cacher-ng-0.7.7/include/header.h 2012-07-23 13:38:14.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/header.h 2012-11-01 20:37:05.000000000 +0000 @@ -70,6 +70,7 @@ void set(eHeadPos, const char *s, size_t len); void set(eHeadPos, off_t nValue); void del(eHeadPos); + inline void copy(const header &src, eHeadPos pos) { set(pos, src.h[pos]); }; inline const char * getCodeMessage() const { return frontLine.length()>9 ? frontLine.c_str()+9 : ""; diff -Nru apt-cacher-ng-0.7.7/include/job.h apt-cacher-ng-0.7.11/include/job.h --- apt-cacher-ng-0.7.7/include/job.h 2012-05-02 21:57:44.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/job.h 2012-11-01 22:35:03.000000000 +0000 @@ -78,16 +78,13 @@ const char * BuildAndEnqueHeader(const FiStatus &fistate, const off_t &nGooddataSize, header& respHead); FiStatus _SwitchToPtItem(const mstring &fileLoc); - void SetErrorResponse(const char * errorLine); + void SetErrorResponse(const char * errorLine, const char *szLocation=NULL); void HandleLocalDownload(const mstring &visPath, const mstring &fsBase, const mstring &fsSubpath); bool ParseRange(); off_t m_nReqRangeFrom, m_nReqRangeTo; - - int m_nRedirRemaining; - bool CreateRedirect(header &respHead); }; #endif diff -Nru apt-cacher-ng-0.7.7/include/meta.h apt-cacher-ng-0.7.11/include/meta.h --- apt-cacher-ng-0.7.7/include/meta.h 2012-06-07 15:24:37.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/meta.h 2012-11-04 17:16:30.000000000 +0000 @@ -409,6 +409,8 @@ return ::time(0); } +#define END_OF_TIME (MYSTD::numeric_limits::max()) + static inline UINT FormatTime(char *buf, const time_t cur) { struct tm tmp; @@ -437,5 +439,7 @@ inline extended_bool(bool val, Textra xtra = defval) : value(val), xdata(xtra) {}; }; +void DelTree(cmstring &what); + #endif // _META_H diff -Nru apt-cacher-ng-0.7.7/include/tcpconnect.h apt-cacher-ng-0.7.11/include/tcpconnect.h --- apt-cacher-ng-0.7.7/include/tcpconnect.h 2012-02-17 19:30:57.000000000 +0000 +++ apt-cacher-ng-0.7.11/include/tcpconnect.h 2012-11-04 16:42:27.000000000 +0000 @@ -11,6 +11,7 @@ #include "sockio.h" class tcpconnect; +class fileitem_base; typedef SHARED_PTR tTcpHandlePtr; class tcpconnect @@ -45,11 +46,11 @@ acfg::tRepoData::IHookHandler *m_pConnStateObserver; - mstring m_lastFileKeyPath; + WEAK_PTR m_lastFile; public: //! @brief Remember the file name belonging to the recently initiated transfer - inline void NameLastFile(cmstring& keypath) { m_lastFileKeyPath=keypath; } + inline void KnowLastFile(WEAK_PTR spRemItem) { m_lastFile = spRemItem; } //! @brief Invalidate (truncate) recently touched file void KillLastFile(); }; diff -Nru apt-cacher-ng-0.7.7/source/acfg.cc apt-cacher-ng-0.7.11/source/acfg.cc --- apt-cacher-ng-0.7.7/source/acfg.cc 2012-08-04 17:49:40.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/acfg.cc 2012-11-04 16:42:27.000000000 +0000 @@ -112,6 +112,7 @@ ,{ "FreshIndexMaxAge", &maxtempdelay, NULL, 10} ,{ "RedirMax", &redirmax, NULL, 10} ,{ "VfileUseRangeOps", &vrangeops, NULL, 10} + ,{ "ResponseFreezeDetectTime", &stucksecs, NULL, 10} ,{ "DirPerms", &dirperms, NULL, 8} ,{ "FilePerms", &fileperms, NULL, 8} @@ -1150,7 +1151,7 @@ } if(redirmax == RESERVED_DEFVAL) - redirmax = forcemanaged ? 0 : 5; + redirmax = forcemanaged ? 0 : ACFG_REDIRMAX_DEFAULT; if (acfg::debug >= LOG_MORE) { @@ -1221,7 +1222,7 @@ time_t ExecutePostponed() { - time_t ret(g_victor.never()), now(time(0)); + time_t ret(END_OF_TIME), now(time(0)); for (tRepMap::iterator it = repoparms.begin(); it != repoparms.end(); ++it) { if (!it->second.m_pHooks) diff -Nru apt-cacher-ng-0.7.7/source/acfg_defaults.cc apt-cacher-ng-0.7.11/source/acfg_defaults.cc --- apt-cacher-ng-0.7.7/source/acfg_defaults.cc 2012-06-07 15:24:37.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/acfg_defaults.cc 2012-11-04 19:38:17.000000000 +0000 @@ -21,7 +21,7 @@ //#define COMPONENT_OPTIONAL "(-[a-z0-9-])" //#define PARANOIASOURCE "(\\.orig|\\.debian)" -string pfilepat(".*(\\.d?deb|\\.rpm|\\.dsc|\\.tar"COMPRLIST"(\\.gpg)?" +string pfilepat(".*(\\.d?deb|\\.rpm|\\.drpm|\\.dsc|\\.tar"COMPRLIST"(\\.gpg)?" "|\\.diff"COMPRLIST"|\\.jigdo|\\.template|changelog|copyright" "|\\.udeb|\\.debdelta|\\.diff/.*\\.gz|(Devel)?ReleaseAnnouncement(\\?.*)?" "|[a-f0-9]+-(susedata|updateinfo|primary|deltainfo).xml.gz" //opensuse, index data, hash in filename @@ -64,7 +64,7 @@ int dnscachetime(1800), dlbufsize(70000), exfailabort(1), exporigin(false), numcores(1), logxff(false), oldupdate(false), recompbz2(false), nettimeout(60), updinterval(0), -forwardsoap(RESERVED_DEFVAL), usewrap(RESERVED_DEFVAL), redirmax(RESERVED_DEFVAL); +forwardsoap(RESERVED_DEFVAL), usewrap(RESERVED_DEFVAL), redirmax(RESERVED_DEFVAL), stucksecs(500); string agentname("Debian Apt-Cacher-NG/" ACVERSION); string remoteport("80"), port("3142"); diff -Nru apt-cacher-ng-0.7.7/source/apt-cacher.cc apt-cacher-ng-0.7.11/source/apt-cacher.cc --- apt-cacher-ng-0.7.7/source/apt-cacher.cc 2012-08-04 18:36:33.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/apt-cacher.cc 2012-11-04 17:16:30.000000000 +0000 @@ -25,6 +25,7 @@ static void _TestCacheDir(); void term_handler(int signum); void log_handler(int signum); +void dump_handler(int signum); void check_algos(); void DispatchAndRunMaintTask(cmstring &cmd, int fd, const char *auth); @@ -121,6 +122,8 @@ sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); sigaction(SIGQUIT, &act, NULL); + + act.sa_handler = &dump_handler; sigaction(SIGUSR2, &act, NULL); act.sa_handler = &log_handler; @@ -185,8 +188,10 @@ acfg::PostProcConfig(); _TestCacheDir(); - //sleep(20); - //g_victor.Start(); + + extern mstring sReplDir; + DelTree(acfg::cacheDirSlash+sReplDir); + conserver::Setup(); if (bForceCleanup) @@ -253,6 +258,7 @@ } cerr << "Cache directory not writable. Check the permissions of " << acfg::cachedir << "!" <id << ".log"; + msg<id << ".log.html"; fd=open(msg.c_str(), O_RDONLY); msg.clear(); if(fd>=0) @@ -243,7 +243,7 @@ return; tSS path(acfg::logdir.length()+24); - path<Unreg(); fi.reset();} goto rep_dlresult; } tHttpUrl origURL; - for(int i=0; i<=REDIRMAX && !bSuccess; ++i) - { - tFileItemPtr fi; - // handle alternative behavior in the first or last cycles - if(0==i && pForcedURL) - { - if(!origURL.SetHttpUrl(pForcedURL)) - return false; - continue; - } - if(i==REDIRMAX) - GOTOREPMSG("too many redirections."); + tFileItemPtr fi; + // handle alternative behavior in the first or last cycles + if (pForcedURL && !origURL.SetHttpUrl(pForcedURL)) + return false; - fi = pItemDelg ? *pItemDelg : fileitem::GetRegisteredFileItem(sFilePathRel); + fi = pItemDelg ? *pItemDelg : fileitem::GetRegisteredFileItem(sFilePathRel, false); - if(!fi) - { - if(nVerbosity) - SendFmt()<<"Checking "<SetupClean()) + GOTOREPMSG("Item busy, cannot reload"); + if (nVerbosity) + SendFmt() << "Downloading " << sFilePathRel << "...\n"; + } + else + { + FiStatus initState = fi->Setup(bIndexFile); + if (FIST_COMPLETE == initState) { - if(!fi->SetupClean()) - GOTOREPMSG("Item busy, cannot reload"); - if(nVerbosity) - SendFmt()<<"Downloading "<\n"; + return true; } - else + if (initState > FIST_COMPLETE) { - FiStatus initState = fi->Setup(bIndexFile); - if(FIST_COMPLETE == initState) - { - SendFmt()<<"Checking "<\n"; - return true; - } - if(initState > FIST_COMPLETE) - { - GOTOREPMSG(fi->GetHeader().frontLine); - } - if(nVerbosity) - SendFmt()<< (bIndexFile?"Checking/Updating " : "Downloading ") - <GetHeader().frontLine); } + if (nVerbosity) + SendFmt() << (bIndexFile ? "Checking/Updating " : "Downloading ") << sFilePathRel + << "...\n"; + } + + if (!FindURL_InitDload(fi, sFilePathRel, sErr, &origURL)) + GOTOREPMSG("Failed to resolve source URL"); - if(i == 0) + if (origURL.sHost.empty()) + GOTOREPMSG("Failed to resolve original URL") + + m_pDlcon->AddJob(fi, origURL); + + m_pDlcon->WorkLoop(); + if (fi->WaitForFinish(NULL) == FIST_COMPLETE) + { + bSuccess = true; + if (nVerbosity) + SendFmt() << "(" << fi->GetTransferCount() / 1024 << "KiB)\n"; + fi->Unreg(); // anyhow, release our pointer + fi.reset(); + } + else + { + if (IsDeprecatedArchFile(sFilePathRel)) { - if( ! FindURL_InitDload(fi, sFilePathRel, sErr, &origURL)) - GOTOREPMSG("Failed to resolve source URL"); + if (nVerbosity == VERB_SHOW) + SendFmt() << "(no longer available)\n"; + m_forcedTrashMap[sFilePathRel] = true; + bSuccess = true; + } + else if (flags.forgiveDlErrors) + { + bSuccess = true; + if (nVerbosity == VERB_SHOW) + SendFmt() << "(ignored)\n"; } - else if(origURL.sHost.empty()) - GOTOREPMSG("Failed to resolve original URL") else - m_pDlcon->AddJob(fi, origURL); - - m_pDlcon->WorkLoop(); - int httpCode; - if(fi->WaitForFinish(&httpCode) != FIST_COMPLETE) - httpCode=555; - - switch(httpCode) - { - case 200: - bSuccess=true; - if(nVerbosity) - SendFmt() << "(" << fi->GetTransferCount() / 1024 << "KiB)\n"; - fi->Unreg(); // anyhow, release our pointer - fi.reset(); - break; - case 301: - case 302: - case 307: - { - header h=fi->GetHeader(); - fi->Unreg(); // anyhow, release our pointer - fi.reset(); - if(h.h[header::LOCATION] && origURL.SetHttpUrl(h.h[header::LOCATION])) - continue; - } - default: - { - if (IsDeprecatedArchFile(sFilePathRel)) - { - if(nVerbosity == VERB_SHOW) - SendFmt() << "(no longer available)\n"; - m_forcedTrashMap[sFilePathRel] = true; - bSuccess = true; - } - else if (flags.forgiveDlErrors) - { - bSuccess = true; - if(nVerbosity == VERB_SHOW) - SendFmt() << "(ignored)\n"; - } - else - GOTOREPMSG(fi->GetHttpMsg()); + GOTOREPMSG(fi->GetHttpMsg()); - fi->Unreg(); // anyhow, release our pointer - fi.reset(); - break; - } - } + fi->Unreg(); // anyhow, release our pointer + fi.reset(); } rep_dlresult: if (bSuccess && bIndexFile) - SetFlags(sFilePathRel).uptodate=true; + SetFlags(sFilePathRel).uptodate = true; UpdateFingerprint(sFilePathRel, -1, NULL, NULL); - if(m_bVerbose || !bSuccess) + if (m_bVerbose || !bSuccess) { - m_bShowControls=true; + m_bShowControls = true; - if(!bSuccess) + if (!bSuccess) { - if(nVerbosity == VERB_SHOW) + if (nVerbosity == VERB_SHOW) { - if(sErr.empty()) + if (sErr.empty()) sErr = "Download error"; SendFmt() << "" << sErr << "\n"; } } - if(nVerbosity == VERB_SHOW) + if (nVerbosity == VERB_SHOW) AddDelCbox(sFilePathRel); } - if(nVerbosity!=VERB_QUIET) + if (nVerbosity != VERB_QUIET) SendChunk("
\n"); return bSuccess; @@ -676,6 +648,7 @@ }; return tFileItemPtr(static_cast(new tHeadOnlyStorage(*this))); } + virtual ~fakeCon() {} }; bool tCacheMan::GetAndCheckHead(cmstring & sTempDataRel, cmstring &sReferencePathRel, @@ -816,7 +789,7 @@ struct keeper { tFileItemPtr pItem; - keeper(const string &path, bool y) { if(y) pItem=fileitem::GetRegisteredFileItem(path); } + keeper(const string &path, bool y) { if(y) pItem=fileitem::GetRegisteredFileItem(path, false); } ~keeper() { if(pItem) pItem->Unreg(); } } src(donorRel, GetFlags(donorRel).uptodate); @@ -1436,7 +1409,7 @@ if (sPureIfileName.length() > 50 && endsWithSzAr(sPureIfileName, ".xml") && sPureIfileName[40] == '-') - return EIDX_SUSEOTHER; + return EIDX_XMLRPMLIST; if (sPureIfileName == "Sources") return EIDX_SOURCES; @@ -1667,8 +1640,8 @@ } } break; - case EIDX_SUSEOTHER: - LOG("SUSE list file, pickup any valid filename ending in .rpm"); + case EIDX_XMLRPMLIST: + LOG("XML based file list, pickup any valid filename ending in .rpm"); while(reader.GetOneLine(sLine)) { if(CheckAbortCondition()) @@ -1678,13 +1651,16 @@ { cmstring tok(split); LOG("testing filename: " << tok); - if(!endsWithSzAr(tok, ".rpm")) - continue; - LOG("RPM basename: " << tok); - info.sFileName = tok; - info.sDirectory = sCurFilesReferenceDirRel; - ret.HandlePkgEntry(info, false); - info.SetInvalid(); + if (endsWithSzAr(tok, ".rpm") + || endsWithSzAr(tok, ".drpm") + || endsWithSzAr(tok, ".srpm")) + { + LOG("RPM basename: " << tok); + info.sFileName = tok; + info.sDirectory = sCurFilesReferenceDirRel; + ret.HandlePkgEntry(info, false); + info.SetInvalid(); + } } } break; diff -Nru apt-cacher-ng-0.7.7/source/caddrinfo.cc apt-cacher-ng-0.7.11/source/caddrinfo.cc --- apt-cacher-ng-0.7.7/source/caddrinfo.cc 2012-08-04 17:49:40.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/caddrinfo.cc 2012-11-04 16:42:27.000000000 +0000 @@ -117,7 +117,7 @@ time_t CAddrInfo::ExpireCache() { lockguard g(&lockDnsCache); - time_t now(time(0)), ret(g_victor.never()); + time_t now(time(0)), ret(END_OF_TIME); for(map::iterator it=mapDnsCache.begin(); it!=mapDnsCache.end(); ) diff -Nru apt-cacher-ng-0.7.7/source/cleaner.cc apt-cacher-ng-0.7.11/source/cleaner.cc --- apt-cacher-ng-0.7.7/source/cleaner.cc 2012-08-04 18:36:33.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/cleaner.cc 2012-11-04 16:42:27.000000000 +0000 @@ -23,7 +23,7 @@ cleaner::cleaner() : m_thr(0), m_bTerminating(false) { - m_nextRun = never(); + m_nextRun = END_OF_TIME; } void cleaner::WorkLoop() @@ -47,12 +47,12 @@ return; continue; } - m_nextRun = never(); + m_nextRun = END_OF_TIME; // can do the work now... } time_t nextTimeProposal = fileitem::DoDelayedUnregAndCheck(); - time_t nextTimeCand = min(nextTimeProposal, never()); + time_t nextTimeCand = min(nextTimeProposal, END_OF_TIME); nextTimeProposal = acfg::ExecutePostponed(); nextTimeCand = min(nextTimeProposal, nextTimeCand); nextTimeProposal = CAddrInfo::ExpireCache(); diff -Nru apt-cacher-ng-0.7.7/source/conn.cc apt-cacher-ng-0.7.11/source/conn.cc --- apt-cacher-ng-0.7.7/source/conn.cc 2012-08-04 17:49:40.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/conn.cc 2012-11-04 09:35:00.000000000 +0000 @@ -217,8 +217,8 @@ ldbg("select con"); struct timeval tv; - tv.tv_sec = 90; - tv.tv_usec = 0; + tv.tv_sec = acfg::nettimeout; + tv.tv_usec = 23; int ready = select(maxfd+1, &rfds, &wfds, NULL, &tv); if(ready == 0) @@ -326,7 +326,17 @@ ldbg("Parsed REQUEST:" << m_pTmpHead->frontLine); ldbg("Rest: " << inBuf.size()); - AddNewJob(m_pTmpHead); + + { + job * j = new job(m_pTmpHead, this); + + j->PrepareDownload(); + m_jobs2send.push_back(j); +#ifdef DEBUG + m_nProcessedJobs++; +#endif + } + m_pTmpHead=NULL; // owned by job now } MYCATCH(bad_alloc&) @@ -453,16 +463,3 @@ sumIn=0; sumOut=0; } - -job* con::AddNewJob(header *pHead) - { - job * j = new job(pHead, this); - - j->PrepareDownload(); - m_jobs2send.push_back(j); -#ifdef DEBUG - m_nProcessedJobs++; -#endif - return j; - -} diff -Nru apt-cacher-ng-0.7.7/source/dlcon.cc apt-cacher-ng-0.7.11/source/dlcon.cc --- apt-cacher-ng-0.7.7/source/dlcon.cc 2012-06-07 22:05:27.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/dlcon.cc 2012-11-05 22:24:59.000000000 +0000 @@ -17,7 +17,7 @@ static const cmstring sGenericError("567 Unknown download error occured"); dlcon::dlcon(bool bManualExecution, string *xff) : - m_bStopASAP(false), m_bManualMode(bManualExecution) + m_bStopASAP(false), m_bManualMode(bManualExecution), m_nDisablePling(0) { LOGSTART("dlcon::dlcon"); m_wakepipe[0] = m_wakepipe[1] = -1; @@ -48,6 +48,7 @@ #define HINT_SWITCH 32 #define EFLAG_LOST_CON 64 #define HINT_KILL_LAST_FILE 128 +#define HINT_TGTCHANGE 256 inline const mstring GetPeerName() { @@ -83,7 +84,9 @@ tDlState m_DlState; - void Init() + int m_nRedirRemaining; + + inline void Init() { m_nRest = 0; m_eReconnectASAP = 0; @@ -93,16 +96,18 @@ m_pStorage->IncDlRefCount(); } - tDlJob(dlcon *p, tFileItemPtr pFi, const tHttpUrl& uri) : + inline tDlJob(dlcon *p, tFileItemPtr pFi, const tHttpUrl& uri, int redirmax) : m_pStorage(pFi), m_pBEdata(NULL), - m_fileUri(uri) + m_fileUri(uri), + m_nRedirRemaining(redirmax) { Init(); } - tDlJob(dlcon *p, tFileItemPtr pFi, const acfg::tRepoData * pBackends, - const MYSTD::string & sPath) : - m_pStorage(pFi), m_pBEdata(pBackends), m_fileUri("", sPath) + inline tDlJob(dlcon *p, tFileItemPtr pFi, const acfg::tRepoData * pBackends, + const MYSTD::string & sPath, int redirmax) : + m_pStorage(pFi), m_pBEdata(pBackends), m_fileUri("", sPath), + m_nRedirRemaining(redirmax) { Init(); } @@ -112,6 +117,46 @@ if (m_pStorage && FIST_COMPLETE != m_pStorage->GetStatus()) m_pStorage->DecDlRefCount(sErrorMsg.empty() ? sGenericError : sErrorMsg); } + + inline bool RewriteSource(const char *pNewUrl) + { + + if (--m_nRedirRemaining <= 0) + { + sErrorMsg = "500 Bad redirection (loop)"; + return false; + } + + if (!pNewUrl) + { + sErrorMsg = "500 Bad redirection (empty)"; + return false; + } + + // it's previous server's job to not send any crap. Just make sure that + // the format is right for further processing here. + mstring s; + for (const char *p = pNewUrl; *p; ++p) + { + if (isspace((unsigned char) *p)) + { + s += "%"; + s += BytesToHexString((uint8_t*) p, 1); + } + else + s += *p; + } + + m_pBEdata = NULL; + m_pCurBackend = NULL; + if (!m_fileUri.SetHttpUrl(s)) + { + sErrorMsg = "500 Bad redirection (invalid URL)"; + return false; + } + + return true; + } // //acfg::tRepoData::IHookHandler * tDlJob::GetConnStateObserver() //{ @@ -127,10 +172,9 @@ //} // needs connectedHost, blacklist, output buffer from the parent, proxy mode? - void AppendRequest(tSS &head, cmstring &xff) + inline void AppendRequest(tSS &head, cmstring &xff) { LOGSTART("tDlJob::AppendRequest"); - bool proxied = ! acfg::proxy_info.sHost.empty(); head << (m_pStorage->m_bHeadOnly ? "HEAD " : "GET "); @@ -270,7 +314,7 @@ * * Process new incoming data and write it down to disk or other receivers. */ - uint_fast8_t ProcessIncomming(acbuf & inBuf) + UINT ProcessIncomming(acbuf & inBuf, bool bOnlyRedirections) { LOGSTART("tDlJob::ProcessIncomming"); if (!m_pStorage) @@ -314,15 +358,32 @@ } ldbg("GOT, parsed: " << h.frontLine); + int st = h.getStatus(); + + // special handling for redirection, see caller ExchangeData() further processing + if(st == 301 || st == 302 || st == 307) + { + if(!acfg::redirmax) + { + ldbg("Internal redirection disabled by administrator, forward it to the user"); + } + else if(RewriteSource(h.h[header::LOCATION])) + return HINT_TGTCHANGE; + else + return EFLAG_JOB_BROKEN; + } + else if(bOnlyRedirections) + return EFLAG_LOST_CON; // any error, just not tgtchange or broken + // explicitly blacklist mirror if key file is missing - if (h.getStatus() >= 400 && m_pBEdata) + if (st >= 400 && m_pBEdata) { for (tStrVecIterConst it = m_pBEdata->m_keyfiles.begin(); it != m_pBEdata->m_keyfiles.end(); ++it) { if (endsWith(m_fileUri.sPath, *it)) { - sErrorMsg = "500 Bad mirror"; + sErrorMsg = "500 Keyfile missing, mirror blacklisted"; return HINT_DISCON | EFLAG_MIRROR_BROKEN; } } @@ -342,7 +403,7 @@ m_DlState = STATE_FINISHJOB; } // the only case where we expect a 304 - else if(h.getStatus() == 304 && acfg::vrangeops == 0) + else if(st == 304 && acfg::vrangeops == 0) { m_pStorage->SetupComplete(); m_DlState = STATE_FINISHJOB; @@ -365,6 +426,8 @@ m_nRest = atoofft(h.h[header::CONTENT_LENGTH]); m_DlState = STATE_PROCESS_DATA; } + + // ok, can pass the data to the file handler h.set(header::XORIG, RemoteUri()); fileitem_base::SuccessWithTransErrorFlag headStored = m_pStorage->DownloadStartedStoreHeader(h, inBuf.rptr(), bHotItem ); if(!headStored && headStored.xdata) @@ -471,6 +534,7 @@ inline bool dlcon::SetupJobConfig(tDlJobPtr &job, mstring *pReasonMsg) { LOGSTART("dlcon::SetupJobConfig"); + // using backends? Find one which is not blacklisted MYMAP, mstring>::const_iterator bliter; @@ -532,12 +596,14 @@ void dlcon::AddJob(tFileItemPtr m_pItem, const acfg::tRepoData *pBackends, const MYSTD::string & sPatSuffix) { - EnqJob(new tDlJob(this, m_pItem, pBackends, sPatSuffix)); + EnqJob(new tDlJob(this, m_pItem, pBackends, sPatSuffix, + m_bManualMode ? ACFG_REDIRMAX_DEFAULT : acfg::redirmax)); } void dlcon::AddJob(tFileItemPtr m_pItem, tHttpUrl hi) { - EnqJob(new tDlJob(this, m_pItem, hi)); + EnqJob(new tDlJob(this, m_pItem, hi, + m_bManualMode ? ACFG_REDIRMAX_DEFAULT : acfg::redirmax)); } void dlcon::SignalStop() @@ -688,6 +754,7 @@ } else if (r == 0) { + dbgline; sErrorMsg = "502 Connection closed"; return EFLAG_LOST_CON; } @@ -716,7 +783,7 @@ { ldbg("Processing job for " << qActive.front()->RemoteUri()); - UINT res = qActive.front()->ProcessIncomming(m_inBuf); + UINT res = qActive.front()->ProcessIncomming(m_inBuf, false); ldbg( "... incoming data processing result: " << res << ", emsg: " << qActive.front()->sErrorMsg); @@ -726,8 +793,9 @@ if (HINT_DONE & res) { + // just in case that server damaged the last response body - con->NameLastFile(qActive.front()->m_pStorage->m_sPathRel); + con->KnowLastFile(WEAK_PTR(qActive.front()->m_pStorage)); qActive.pop_front(); if (HINT_DISCON & res) @@ -746,15 +814,30 @@ LOG("Extract more responses"); continue; } - else // error handling, pass to main loop - { - - if(HINT_KILL_LAST_FILE & res) - con->KillLastFile(); - setIfNotEmpty(sErrorMsg, qActive.front()->sErrorMsg); - return res; + if (HINT_TGTCHANGE & res) + { + /* target was rewritten by the job handler, but there might be more redirections + in the pipe which we can analyze and use ASAP to avoid later requests. + Process as much as we can (carefully). */ + tDljQueue::iterator it = qActive.begin(); + for(++it; it != qActive.end(); ++it) + { + UINT rr = (**it).ProcessIncomming(m_inBuf, true); + if( HINT_TGTCHANGE != rr ) + { + m_nDisablePling=30; + return (HINT_TGTCHANGE|HINT_DISCON); + } + } + return HINT_TGTCHANGE; } + + // else case: error handling, pass to main loop + if(HINT_KILL_LAST_FILE & res) + con->KillLastFile(); + setIfNotEmpty(sErrorMsg, qActive.front()->sErrorMsg); + return res; } return HINT_DONE; // input buffer consumed } @@ -905,7 +988,7 @@ && ( cjob->GetPeerName() != con->GetHostname() || cjob->GetPeerPort() != con->GetPort()) ) { - LOG("host mismatch, stop requesting"); + LOG("host mismatch, stop sending requesting for now"); bStopRequesting=true; break; } @@ -915,6 +998,12 @@ LOG("request added to buffer"); inpipe.push_back(cjob); m_qNewjobs.pop_front(); + + if(m_nDisablePling >0) + { + bStopRequesting=true; + --m_nDisablePling; + } } } } @@ -933,19 +1022,22 @@ loopRes=ExchangeData(sErrorMsg, con, inpipe); ldbg("loopRes: "<< loopRes); - // no matter what happened, that stop flag is now irrelevant - if(inpipe.empty()) - { - dbgline; - bStopRequesting=false; - } + if (inpipe.empty()) + { + // all requests have been processed (client done, or pipeline stall, who cares) + dbgline; - if(0==(loopRes&(HINT_DISCON|EFLAG_JOB_BROKEN|EFLAG_LOST_CON|EFLAG_MIRROR_BROKEN)) - && con && inpipe.empty()) // no error bits set, not busy -> recycle it when idle - { - dbgline; - tcpconnect::RecycleIdleConnection(con); - continue; + // no matter what happened, that stop flag is now irrelevant + bStopRequesting = false; + + // no error bits set, not busy -> this connection is still good, recycle properly + UINT all_err = HINT_DISCON | EFLAG_JOB_BROKEN | EFLAG_LOST_CON | EFLAG_MIRROR_BROKEN; + if (con && !(loopRes & all_err)) + { + dbgline; + tcpconnect::RecycleIdleConnection(con); + continue; + } } if( (HINT_DISCON|EFLAG_LOST_CON) & loopRes) @@ -954,6 +1046,14 @@ con.reset(); } + if ( loopRes & HINT_TGTCHANGE ) + { + // short queue has been rewritten, needs to play back everything + // if not reset above -> should be in good shape + tcpconnect::RecycleIdleConnection(con); + goto move_jobs_back_to_q; + } + if ((EFLAG_LOST_CON & loopRes) && !inpipe.empty()) { // disconnected by OS... give it a chance, or maybe not... @@ -1008,10 +1108,10 @@ setLockGuard; CleanRunning(m_qNewjobs); } - } - // if the job was not dropped, push it back into the task queue + move_jobs_back_to_q: + // for the jobs that were not finished and/or dropped, move them back into the task queue { setLockGuard; m_qNewjobs.insert(m_qNewjobs.begin(), inpipe.begin(), inpipe.end()); diff -Nru apt-cacher-ng-0.7.7/source/expiration.cc apt-cacher-ng-0.7.11/source/expiration.cc --- apt-cacher-ng-0.7.7/source/expiration.cc 2012-07-23 13:38:14.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/expiration.cc 2012-11-04 19:38:17.000000000 +0000 @@ -185,7 +185,7 @@ if(GetFlags(sPathRel).parseignore) { SendFmt() << sPathRel << ": incorrect or obsolete contents but marked as " - "alternative version of another index file (ignoring)"; + "alternative version of another index file (ignoring)"; } else { @@ -560,7 +560,7 @@ void expiration::PurgeMaintLogs() { - tStrDeq logs = ExpandFilePattern(acfg::logdir + SZPATHSEP"maint_*.log"); + tStrDeq logs = ExpandFilePattern(acfg::logdir + SZPATHSEP"maint_*.log*"); if (logs.size() > 2) SendChunk( "Found required cleanup tasks: purging maintanence logs...
\n"); diff -Nru apt-cacher-ng-0.7.7/source/fileitem.cc apt-cacher-ng-0.7.11/source/fileitem.cc --- apt-cacher-ng-0.7.7/source/fileitem.cc 2012-08-04 17:49:40.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/fileitem.cc 2012-12-16 18:56:41.000000000 +0000 @@ -16,6 +16,7 @@ using namespace MYSTD; #define MAXTEMPDELAY acfg::maxtempdelay // 27 +mstring sReplDir("_altStore" SZPATHSEP); header const & fileitem_base::GetHeaderUnlocked() { @@ -30,7 +31,7 @@ return m_head.frontLine; } -time_t fileitem_base::m_nEarliestExpiration(cleaner::never()); +time_t fileitem_base::m_nEarliestExpiration(END_OF_TIME); fileitem_base::fileitem_base(MYSTD::string sPath) : condition(), @@ -44,7 +45,8 @@ m_filefd(-1), m_nDlRefsCount(0), m_status(FIST_FRESH), - m_nTimeExpireAt(0) + m_nTimeExpireAt(0), + m_nTimeDlDone(END_OF_TIME) { // FIXME: this sucks, belongs into the fileitem @@ -113,18 +115,18 @@ m_filefd(-1), m_nDlRefsCount(0), m_status(FIST_FRESH), - m_nTimeExpireAt(0) + m_nTimeExpireAt(0), + m_nTimeDlDone(END_OF_TIME) { } fileitem_base::~fileitem_base() { - setLockGuard; + //setLockGuard; // m_head.clear(); checkforceclose(m_filefd); } - void fileitem_base::IncDlRefCount() { setLockGuard; @@ -344,7 +346,7 @@ #define SET_FRONTLINE(ret, x) ret="HTTP/1.1 "; ret+=x; #define SETERROR(x) { m_bAllowStoreData=false; SET_FRONTLINE(m_head.frontLine, x); \ m_head.set(header::XORIG, h.h[header::XORIG]); \ - m_status=FIST_LOCALERROR; _LogWithErrno(x, m_sPathRel); } + m_status=FIST_LOCALERROR; m_nTimeDlDone=GetTime(); _LogWithErrno(x, m_sPathRel); } #define SETERRORKILLFILE(x) { SETERROR(x); goto kill_file; } #define BOUNCE(x) { SETERROR(x); return false; } @@ -506,24 +508,18 @@ { // -> kill cached file ASAP m_bAllowStoreData=false; - m_head.set(header::XORIG, h.h[header::XORIG]); + m_head.copy(h, header::XORIG); SETERRORKILLFILE("503 Server disagrees on file size, cleaning up"); } break; -// use the Location header and be sure the server does not mess around with Location and other codes - case 301: - case 302: - case 303: - case 307: - m_head.set(header::LOCATION, h.h[header::LOCATION]); - // fall-through default: m_head.type=header::ANSWER; - m_head.set(header::XORIG, h.h[header::XORIG]); + m_head.copy(h, header::XORIG); + m_head.copy(h, header::LOCATION); if(bForcedRestart) { - // got a redirection from another mirror? or an error? cannot handle it yet. - // XXX: concept change required + // got an error from the replacement mirror? cannot handle it properly + // because some job might already have started returning the data USRDBG(0, "Cannot restart, HTTP code: " << serverStatus); BOUNCE(h.getCodeMessage()); } @@ -648,6 +644,7 @@ unlink(sHeadPath.c_str()); m_status=FIST_LOCALERROR; + m_nTimeDlDone=GetTime(); return false; } @@ -675,6 +672,7 @@ else { m_status = FIST_COMPLETE; + m_nTimeDlDone=GetTime(); if (acfg::debug & LOG_MORE) aclog::misc(tSS() << "Download of " << m_sPathRel << " finished"); @@ -703,6 +701,7 @@ aclog::errnoFmter efmt("HTTP/1.1 503 "); m_head.frontLine = efmt; m_status=FIST_LOCALERROR; + m_nTimeDlDone=GetTime(); _LogWithErrno(efmt.c_str(), m_sPathRel); return false; } @@ -720,159 +719,165 @@ return true; } - -struct tFileRefEntry -{ - int nRefCount; - tFileItemPtr ref; - bool m_bNeedsDelayedExpiration; - tFileRefEntry() : nRefCount(0), m_bNeedsDelayedExpiration(false) { }; -}; -static MYMAP mapItems; +//typedef MYSTD::pair tFileRefEntry; +//typedef MYSTD::multimap tFiGlobMap; +static tFiGlobMap mapItems; lockable mapLck; +fileitem::fileitem(cmstring &s) : fileitem_with_storage(s), m_globRef(mapItems.end()) +{}; + void fileitem::Unreg() { LOGSTART("fileitem::Unreg"); - lockguard managementLock(mapLck); - - MYMAP::iterator entry = mapItems.find(m_sPathRel); - if(mapItems.end() == entry) - { - aclog::err("INTERNAL ERROR, attempt to unregister non-existing download item"); - return; - } - if(entry->second.ref.get() != this) - { - aclog::err("INTERNAL ERROR, doppelganger running amok"); - return; - } - entry->second.nRefCount--; - if(entry->second.nRefCount<=0) - { - LOG("*this is last entry, deleting dl/fi mapping"); - setLockGuard; - m_status=FIST_ERRNOUSER; - notifyAll(); - mapItems.erase(entry); - } + lockguard managementLock(mapLck); + setLockGuard; + + if(m_globRef == mapItems.end()) + { + // not globally registered, just ignore + return; + } + + m_globRef->second.first--; + if (m_globRef->second.first <= 0) + { + // use the delay timer only if late expiration is needed and it isn't expired already + if (m_nTimeExpireAt && GetTime() < m_nTimeExpireAt) + { + m_nEarliestExpiration = MYSTD::min(m_nTimeExpireAt, m_nEarliestExpiration); + g_victor.RunLatestAt(m_nEarliestExpiration); + return; + } + + LOG("*this is last entry, deleting dl/fi mapping"); + m_status = FIST_ERRNOUSER; + notifyAll(); + mapItems.erase(m_globRef); + m_globRef = mapItems.end(); + } } -tFileItemPtr fileitem::GetRegisteredFileItem(MYSTD::string sPathKey) + +tFileItemPtr fileitem::GetRegisteredFileItem(cmstring &sPathKey, bool bConsiderAltStore) { LOGSTART2s("fileitem::GetFileItem", sPathKey); - tFileItemPtr p; + MYTRY { - p.reset(new fileitem(sPathKey)); // let it create real target path + fileitem *p = new fileitem(sPathKey); // need one because it will convert the path string to the proper value needed as key + tFileItemPtr sp(p); lockguard lockGlobalMap(mapLck); - map::iterator it=mapItems.find(p->m_sPathRel); + tFiGlobMap::iterator it=mapItems.find(p->m_sPathRel); if(it!=mapItems.end()) { - it->second.nRefCount++; + if (bConsiderAltStore) + { + // detect items that got stuck somehow + time_t now(GetTime()); + time_t extime(now - acfg::stucksecs); + if (it->second.second.get()->m_nTimeDlDone < extime) + { + // try to find its sibling which is in good state? + for (; it!=mapItems.end() && it->first == p->m_sPathRel; ++it) + { + if (it->second.second.get()->m_nTimeDlDone >= extime) + { + it->second.first++; + LOG("Sharing an existing REPLACEMENT file item"); + return it->second.second; + } + } + // ok, then create a modded name version in the replacement directory + mstring sKeyPathOrig(p->m_sPathRel); + replaceChars(p->m_sPathRel, "/\\", '_'); + p->m_sPathRel.insert(0, sReplDir + ltos(getpid()) + "_" + ltos(now))+"_"; + p->m_sPathAbs = acfg::cacheDirSlash+p->m_sPathRel; + + LOG("Registering a new REPLACEMENT file item..."); + p->m_globRef = mapItems.insert(make_pair(sKeyPathOrig, tFileRefEntry(1,sp))); + return sp; + } + } LOG("Sharing existing file item"); - return it->second.ref; + it->second.first++; + return it->second.second; } - // ok, keep the new item - - LOG("Created new file item"); - tFileRefEntry & entry = mapItems[p->m_sPathRel]; - LOG("Registering new file item..."); - if(!entry.ref) - { - entry.ref=p; - entry.nRefCount++; - LOG("New reference count: " << entry.nRefCount); - } - return entry.ref; + LOG("Registering the NEW file item..."); + p->m_globRef = mapItems.insert(make_pair(p->m_sPathRel, tFileRefEntry(1,sp))); + return sp; } MYCATCH(MYSTD::bad_alloc&) { - p.reset(); } - return p; + return tFileItemPtr(); } +// make the fileitem globally accessible bool fileitem::RegisterFileItem(SHARED_PTR spCustomFileItem) - { - if (!spCustomFileItem || spCustomFileItem->m_sPathRel.empty()) - return false; - - lockguard lockGlobalMap(mapLck); - - map::iterator it = mapItems.find(spCustomFileItem->m_sPathRel); - if (it != mapItems.end()) - return false; // conflict, another agent is already active - - tFileRefEntry & entry = mapItems[spCustomFileItem->m_sPathRel]; - entry.ref = spCustomFileItem; - entry.nRefCount = 1; - - return true; - } - -// Opposite of GetFileItem, unregisters existing file item now or as soon as possible. -void fileitem::UnregOneASAP(tFileItemPtr pItem) { - if (!pItem) - return; - // download never started or no delayed expiration needed -> release now. Or expired long ago. - if (!pItem->m_nTimeExpireAt || GetTime() >= pItem->m_nTimeExpireAt) - { - pItem->Unreg(); - return; - } + if (!spCustomFileItem || spCustomFileItem->m_sPathRel.empty()) + return false; + lockguard lockGlobalMap(mapLck); - MYMAP::iterator entry = mapItems.find(pItem->m_sPathRel); - // not existing? -> not for us. Still enough references? -> let this one go - if (mapItems.end() == entry || entry->second.nRefCount>1) - { - lockGlobalMap.unLock(); - pItem->Unreg(); - return; - } - // now the item's refCount is by one higher then it should be, set the flag instead of -- - entry->second.m_bNeedsDelayedExpiration=true; - m_nEarliestExpiration = MYSTD::min(pItem->m_nTimeExpireAt, m_nEarliestExpiration); - g_victor.RunLatestAt(m_nEarliestExpiration); + if(ContHas(mapItems, spCustomFileItem->m_sPathRel)) + return false; // conflict, another agent is already active + spCustomFileItem->m_globRef = mapItems.insert(make_pair(spCustomFileItem->m_sPathRel, + tFileRefEntry(1, spCustomFileItem))); + + return true; } + // this method is supposed to be awaken periodically and detect items with ref count manipulated by // the request storm prevention mechanism. Items shall be be dropped after some time if no other // thread but us is using them. time_t fileitem::DoDelayedUnregAndCheck() { lockguard lockGlobalMap(mapLck); - MYMAP::iterator it, here; + tFiGlobMap::iterator it, here; time_t now=GetTime(); if(nowsecond.m_bNeedsDelayedExpiration) + + // not needed or not ready yet + if (!here->second.second->m_nTimeExpireAt) continue; - // still active or young item? skip, but find the earliest time to expire - if(now < here->second.ref->m_nTimeExpireAt || here->second.nRefCount>1) - { - m_nEarliestExpiration=min(here->second.ref->m_nTimeExpireAt, m_nEarliestExpiration); + + m_nEarliestExpiration=min(here->second.second->m_nTimeExpireAt, m_nEarliestExpiration); + + if (now < here->second.second->m_nTimeExpireAt) continue; + + // became busy again? Ignore this entry, it's new master will take care of the deletion ASAP + if(here->second.first>0) + continue; + + // ok, unused and delay is over. Destroy with the same sequence as Unreg does, care about + // live time. + tFileItemPtr local_ptr(here->second.second); + fileitem *p = static_cast(local_ptr.get()); + { + lockguard g(p); + p->m_status = FIST_ERRNOUSER; + p->m_globRef = mapItems.end(); + mapItems.erase(here); } - // ok, unused and delay is over. Destroy with the same sequence as Unreg... does. - lockguard g(here->second.ref.get()); - here->second.ref->m_status=FIST_ERRNOUSER; - here->second.ref->notifyAll(); - g.unLock(); - mapItems.erase(here); + // p still alive, through local_ptr or external references to it + p->notifyAll(); + } return m_nEarliestExpiration; } @@ -909,6 +914,49 @@ #endif } +void dump_handler(int) { + fileitem::dump_status(); +} + +void fileitem::dump_status() +{ + tSS fmt; + aclog::err("File descriptor table:\n", NULL); + for(tFiGlobMap::iterator it=mapItems.begin(); it!=mapItems.end(); ++it) + { + fmt.clear(); + fmt << "FREF: " << it->first << " [" << it->second.first << "]:\n"; + if(! it->second.second) + { + fmt << "\tBAD REF!\n"; + continue; + } + else + { + fmt << "\t" << it->second.second->m_sPathRel + << "\n\t" << it->second.second->m_sPathAbs + << "\n\tPos.dl.agents: " << it->second.second->m_nDlRefsCount + << "\n\tState: " << it->second.second->m_status + << "\n\tPos: " << it->second.second->m_nIncommingCount << " , " + << it->second.second->m_nRangeLimit << " , " + << it->second.second->m_nSizeChecked << " , " + << it->second.second->m_nSizeSeen + << "\n\tExp: " << it->second.second->m_nEarliestExpiration << " , " + << it->second.second->m_nTimeExpireAt << "\n\n"; + } + aclog::err(fmt.c_str(), NULL); + } +} + +fileitem_with_storage::~fileitem_with_storage() +{ + if(startsWith(m_sPathRel, sReplDir)) + { + unlink(m_sPathAbs.c_str()); + unlink((m_sPathAbs+".head").c_str()); + } +} + #endif // MINIBUILD FiStatus fileitem_base::WaitForFinish(int *httpCode) @@ -920,3 +968,4 @@ *httpCode=m_head.getStatus(); return m_status; } + diff -Nru apt-cacher-ng-0.7.7/source/filereader.cc apt-cacher-ng-0.7.11/source/filereader.cc --- apt-cacher-ng-0.7.7/source/filereader.cc 2012-02-20 21:14:22.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/filereader.cc 2012-11-01 20:37:05.000000000 +0000 @@ -596,53 +596,6 @@ print "\n"; */ -#define _inv MAX_VAL(uint_fast16_t) - -uint_fast16_t hexmap[] = { - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - 0,1,2,3,4,5,6,7,8,9,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,10,11,12,13,14,15,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,10,11,12,13,14,15,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, - _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv - }; - -bool CsAsciiToBin(const char *a, uint8_t b[], unsigned short binLength) -{ - const unsigned char *uA = (const unsigned char*) a; - for(int i=0; i15 || hexmap[uA[i*2+1]] > 15) - return false; - b[i]=hexmap[uA[i*2]] * 16 + hexmap[uA[i*2+1]]; - } - return true; -} - -string BytesToHexString(const uint8_t sum[], unsigned short lengthBin) -{ - string sRet; - char hexmap[] = - { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', - 'e', 'f' }; - for (int i=0; i>4]; - sRet+=(char) hexmap[sum[i]&0xf]; - } - - return sRet; -} /* bool filereader::CheckMd5Sum(const string & sFileName, const string & sReference) diff -Nru apt-cacher-ng-0.7.7/source/job.cc apt-cacher-ng-0.7.11/source/job.cc --- apt-cacher-ng-0.7.7/source/job.cc 2012-07-23 13:38:14.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/job.cc 2012-11-04 16:42:27.000000000 +0000 @@ -186,6 +186,7 @@ // never used to store data SuccessWithTransErrorFlag DownloadStartedStoreHeader(const header &, const char *, bool) {return false;}; bool StoreFileData(const char *, unsigned int) {return false;}; + header & HeadRef() { return m_head; } }; ssize_t sendfile_generic(int out_fd, int in_fd, off_t *offset, size_t count) @@ -249,8 +250,8 @@ m_nAllDataCount(0), m_nChunkRemainingBytes(0), m_type(rechecks::FILE_INVALID), - m_nReqRangeFrom(-1), m_nReqRangeTo(-1), - m_nRedirRemaining(acfg::redirmax) + m_nReqRangeFrom(-1), m_nReqRangeTo(-1) +//,m_nRedirRemaining(acfg::redirmax) { LOGSTART2("job::job", "job creating, " << m_pReqHead->frontLine << " and this: " << uintptr_t(this)); } @@ -269,9 +270,8 @@ m_nAllDataCount, bErr); checkforceclose(m_filefd); - - fileitem::UnregOneASAP(m_pItem); - + if(m_pItem) + m_pItem->Unreg(); delete m_pReqHead; } @@ -493,43 +493,6 @@ return false; } -bool job::CreateRedirect(header & respHead) -{ - int st = respHead.getStatus(); - if (m_nRedirRemaining <= 0 || (st != 301 && st != 302 && st != 307) // redir limit reached or not redirect - || !respHead.h[header::LOCATION] || !respHead.h[header::LOCATION][0]) // or nonsense - return false; - - MYTRY - { - header *xheader = new header(*m_pReqHead); - // it's previous server's job to not send any crap. Just make sure that the format is right... - xheader->frontLine = string("GET "); - for (const char *p = respHead.h[header::LOCATION]; *p; ++p) - { - if (isspace((unsigned char) *p)) - { - xheader->frontLine += "%"; - xheader->frontLine += BytesToHexString((uint8_t*) p, 1); - } - else - xheader->frontLine += *p; - } - xheader->frontLine += " HTTP/1.1"; - job *pReplacement = m_pParentCon->AddNewJob(xheader); - if (!pReplacement) - return false; - pReplacement->m_nRedirRemaining = m_nRedirRemaining - 1; - } - MYCATCH(...) - { - return false; - } - return true; -} - - - void job::PrepareDownload() { LOGSTART("job::PrepareDownload"); @@ -667,7 +630,7 @@ bForceFreshnessChecks = ( ! acfg::offlinemode && m_type == FILE_INDEX); - m_pItem=fileitem::GetRegisteredFileItem(m_sFileLoc); + m_pItem=fileitem::GetRegisteredFileItem(m_sFileLoc, bForceFreshnessChecks); } MYCATCH(MYSTD::out_of_range&) // better safe... @@ -823,7 +786,7 @@ m_sOrigUrl=h.h[header::XORIG]; // must be something like "HTTP/1.1 403 Forbidden" if(h.frontLine.length()>9 && h.getStatus()!=200) - SetErrorResponse(h.frontLine.c_str()+9); + SetErrorResponse(h.frontLine.c_str()+9, h.h[header::LOCATION]); else // good ungood? confused somewhere?! SetErrorResponse("500 Unknown error"); return R_AGAIN; @@ -874,9 +837,6 @@ m_state=STATE_SEND_BUFFER; m_backstate=STATE_HEADER_SENT; // could be changed while creating header - if(CreateRedirect(respHead)) - return R_DONE; - const char *szErr = BuildAndEnqueHeader(fistate, nGoodDataSize, respHead); if(szErr) THROW_ERROR(szErr); USRDBG(REPLEVEL, "Prepared response header for user: \n" << m_sendbuf ); @@ -1076,7 +1036,6 @@ bool bFreshnessForced = (m_type != rechecks::FILE_INDEX || m_pReqHead->h[header::ACNGFSMARK] || !pIfmo); - // this plays well with lighttpd <-> Apt, but may not work with the alternative time format struct tm tm1, tm2; bool bIfModSeenAndChecked=false; if(pIfmo && header::ParseDate(pIfmo, &tm1) && header::ParseDate(pLastMo, &tm2)) @@ -1192,10 +1151,12 @@ } -void job::SetErrorResponse(const char * errorLine) +void job::SetErrorResponse(const char * errorLine, const char *szLocation) { LOGSTART2("job::SetErrorResponse", errorLine); + if(m_pItem) + m_pItem->Unreg(); class erroritem: public tGeneratedFitemBase { @@ -1213,12 +1174,10 @@ } }; - tFileItemPtr pNew(static_cast(new erroritem("noid", errorLine))); - - fileitem::UnregOneASAP(m_pItem); - + erroritem *p = new erroritem("noid", errorLine); + p->HeadRef().set(header::LOCATION, szLocation); + tFileItemPtr pNew(static_cast(p)); m_pItem=pNew; - //aclog::err(tSS() << "fileitem is now " << uintptr_t(m_pItem.get())); m_state=STATE_SEND_MAIN_HEAD; } diff -Nru apt-cacher-ng-0.7.7/source/lockable.cc apt-cacher-ng-0.7.11/source/lockable.cc --- apt-cacher-ng-0.7.7/source/lockable.cc 2011-02-06 18:57:04.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/lockable.cc 2012-11-04 16:42:27.000000000 +0000 @@ -41,7 +41,7 @@ void condition::notifyAll() { //ldbg("broadcast " << (uint64_t) &__cond); - pthread_cond_broadcast(&__cond); + pthread_cond_broadcast(&__cond); } void condition::notify() { diff -Nru apt-cacher-ng-0.7.7/source/meta.cc apt-cacher-ng-0.7.11/source/meta.cc --- apt-cacher-ng-0.7.7/source/meta.cc 2012-05-23 10:57:10.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/meta.cc 2012-11-01 20:37:05.000000000 +0000 @@ -426,4 +426,55 @@ } */ + + #endif + + +#define _inv MAX_VAL(uint_fast16_t) + +uint_fast16_t hexmap[] = { + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + 0,1,2,3,4,5,6,7,8,9,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,10,11,12,13,14,15,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,10,11,12,13,14,15,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv, + _inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv,_inv + }; + +bool CsAsciiToBin(const char *a, uint8_t b[], unsigned short binLength) +{ + const unsigned char *uA = (const unsigned char*) a; + for(int i=0; i15 || hexmap[uA[i*2+1]] > 15) + return false; + b[i]=hexmap[uA[i*2]] * 16 + hexmap[uA[i*2+1]]; + } + return true; +} + +string BytesToHexString(const uint8_t sum[], unsigned short lengthBin) +{ + string sRet; + char hexmap[] = + { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', + 'e', 'f' }; + for (int i=0; i>4]; + sRet+=(char) hexmap[sum[i]&0xf]; + } + + return sRet; +} diff -Nru apt-cacher-ng-0.7.7/source/tcpconnect.cc apt-cacher-ng-0.7.11/source/tcpconnect.cc --- apt-cacher-ng-0.7.7/source/tcpconnect.cc 2012-08-04 17:49:40.000000000 +0000 +++ apt-cacher-ng-0.7.11/source/tcpconnect.cc 2012-11-04 16:42:27.000000000 +0000 @@ -167,7 +167,7 @@ m_pConnStateObserver->JobRelease(); m_pConnStateObserver=NULL; - m_lastFileKeyPath.clear(); + m_lastFile.reset(); forceShutdownClose(m_conFd); } @@ -326,21 +326,16 @@ ++it; } - return spareConPool.empty() ? g_victor.never() : GetTime()+TIME_SOCKET_EXPIRE_CLOSE/4+1; + return spareConPool.empty() ? END_OF_TIME : GetTime()+TIME_SOCKET_EXPIRE_CLOSE/4+1; } void tcpconnect::KillLastFile() { #ifndef MINIBUILD - if (m_lastFileKeyPath.empty()) - return; - - tFileItemPtr p(fileitem::GetRegisteredFileItem(m_lastFileKeyPath)); + tFileItemPtr p = m_lastFile.lock(); if (!p) return; - p->SetupClean(true); - p->Unreg(); #endif }