diff -Nru apt-cacher-1.7.15/apt-cacher-cleanup.pl apt-cacher-1.7.16/apt-cacher-cleanup.pl --- apt-cacher-1.7.15/apt-cacher-cleanup.pl 2017-03-14 16:55:02.000000000 +0000 +++ apt-cacher-1.7.16/apt-cacher-cleanup.pl 2017-11-08 09:14:29.000000000 +0000 @@ -167,17 +167,9 @@ } sub get { - my ($file, $use_url) = @_; - my $path_info; - # use path if stored in cached header/complete file - if($use_url && (my $url = get_original_url($file))) { - $path_info= $url; - } - else { - $path_info=$file; - $path_info=~s/^/\//; - $path_info=~s/_/\//g; - } + my ($file) = @_; + + my $path_info = get_original_url($file); defined(my $refresh_pid = open(my $fh, '|-')) || die "Failed to open refresh pipe: $!"; if ($refresh_pid){ printmsg "Get $path_info\n"; @@ -513,6 +505,39 @@ return; } +# Parse InRelease for by-hash data +# Return arrayref of relevant (Sources, Packages, diff_Index) by-hash filenames +sub by_hash { + my ($file) = @_; + my @ret; + + open(my $fh, '<', $file) or + die "Failed to open $file: $!"; + + while (<$fh>) { + # Loop through file and only parse it fully if Acquire-By-Hash is set + /^Acquire-By-Hash: yes$/ && do { + my %parsed; + extract_sums($file, $fh, \%parsed) || die("Error extracting by-hash data from $file\n"); + foreach my $index_file (keys %parsed) { + next unless $index_file =~ /_(?:Sourc|Packag)es(?:\.(?:gz|xz|bz2|diff_Index))?/; + my $href = hashify(\$parsed{$index_file}); + while (my ($k,$v) = each %$href) { + next if $k eq 'size'; + $k = uc $k; + $k .= 'Sum' if $k eq 'MD5'; + if ((my $possible = $index_file) =~ s/[^_]+$/by-hash_${k}_$v/) { + push (@ret, $possible); + } + } + } + last; + } + } + return \@ret; +} + + ############################################################################# # Manipulate checksum database if (@db_mode || $db_recover){ @@ -652,27 +677,42 @@ } if (!$offline) { - # Try to ensure corresponding Packages/Sources is present for each diff_Index - # and Release for each Packages/Sources - my %missing; - CHECKFILE: - foreach (glob('*diff_Index *{Packages,Sources}{,.gz,.xz,.bz2}')) { - my $file = $_; - if (s/\.diff_Index$/{,.gz,.xz,.bz2}/) { - printmsg "Checking for $_ for $file\n"; - } - elsif (s/(?:dists_[^_]+_(?:updates_)?\K(?:[^_]+_){2})?(?:Packages|Sources)(?:\.(?:bz2|(?:x|g)z))?$/{In,}Release/) { - printmsg "checking for $_ for $file\n"; - } + # Ensure corresponding Packages/Sources is present for each + # diff_Index/by-hash and Release for each Packages/Sources. + # Do separate passes as diff_Index or by-hash may cause download of new + # Packages or Sources. + foreach ('*diff_{Index,by-hash_*}', '*{Packages,Sources}{,.gz,.xz,.bz2}') { + my %missing; + CHECKFILE: foreach (glob) { - if ( -f $_ ) { - printmsg "Found $_\n"; - next CHECKFILE; - } + my $file = $_; + s/\.diff_(?:Index$|(?:by-hash_(?i:MD5SUM_[0-9a-f]{32}|SHA1_[0-9a-f]{40}|SHA256_[0-9a-f]{64})))/{.xz,.gz,.bz2,}/ + or + s/(?:dists_[^_]+_(?:updates_)?\K(?:[^_]+_){2})?(?:Packages|Sources)(?:\.(?:bz2|(?:x|g)z))?$/{In,}Release/; + printmsg "checking for $_ as index for $file\n"; + foreach (glob) { + if ( -f $_ ) { + printmsg "Found $_\n"; + next CHECKFILE; + } + } + push @{$missing{$_}}, $file; + } + + MISSING: + foreach (keys %missing) { + foreach (glob) { + my $resp; + get($_); + $resp = read_header("../headers/$_") if -f "../headers/$_" and -f $_; + next MISSING if $resp && $resp->is_success; + printmsg "$_ not available upstream\n"; + } + warn "$_ not available upstream. \nRemove the following files manually if they are no longer significant and you want to clean cached packages they reference:-\n"; + warn " $_\n" foreach @{$missing{$_}}; + die "Exiting to prevent deletion of cache contents.\n"; } - $missing{(glob)[-1]} = 1; } - get($_) foreach keys %missing; } @@ -697,6 +737,16 @@ } } + # Handle by-hash index files + if ($file =~ /InRelease$/) { + foreach my $candidate (@{by_hash($file)}) { + if (-f $candidate) { + printmsg "Cached file $candidate is current by-hash file referenced in $file. Adding to list for parsing\n"; + $valid{$candidate} = 1; + } + } + } + # Remove obsolete Release and Release.gpg if ((my $obsolete = $file) =~ s/InRelease/Release/) { next unless exists $valid{$obsolete}; @@ -773,10 +823,10 @@ unlink $_, "../headers/$_" unless $sim_mode; printmsg "Removing expired changelog: $_ and company...\n"; } - elsif (is_file_type('installer', get_original_url($_))) { # APT/installer files - # Verify + elsif (is_file_type('installer', get_original_url($_))) { # APT/installer/by-hash files + # Verify still present upstream printmsg "Validating $_\n"; - get($_,1) unless $sim_mode; + get($_) unless $sim_mode; } elsif (is_file_type('package', $_) or is_file_type('pdiff', get_original_url($_))) { # Package and pdiff files diff -Nru apt-cacher-1.7.15/debian/changelog apt-cacher-1.7.16/debian/changelog --- apt-cacher-1.7.15/debian/changelog 2017-03-21 09:52:04.000000000 +0000 +++ apt-cacher-1.7.16/debian/changelog 2017-11-08 09:16:20.000000000 +0000 @@ -1,3 +1,18 @@ +apt-cacher (1.7.16) unstable; urgency=medium + + * Update to Standards version 4.1.1 (no changes). + * Ensure relevant base index file is present for diff_by-hash_* files + (closes: #878169). + * When fetching missing index files, exit with a manual resolution + message if no upstream version can be found. + * Add Ubuntu 17.10 release name 'artful'. + * Add Ubuntu 18.04 release name 'bionic'. + * Correctly determine original URL from filename for Translation-??_?? + files. + * apt-cacher-cleanup/pl: remove unused $use_url parameter from get(). + + -- Mark Hindley Wed, 08 Nov 2017 09:16:20 +0000 + apt-cacher (1.7.15) unstable; urgency=medium * Prevent HTTP response splitting with encoded newlines in request. diff -Nru apt-cacher-1.7.15/debian/control apt-cacher-1.7.16/debian/control --- apt-cacher-1.7.15/debian/control 2017-02-21 11:09:09.000000000 +0000 +++ apt-cacher-1.7.16/debian/control 2017-11-08 09:14:37.000000000 +0000 @@ -4,7 +4,7 @@ Maintainer: Mark Hindley Uploaders: Eduard Bloch Build-Depends: debhelper (>= 8.1.0~), po-debconf -Standards-Version: 3.9.8 +Standards-Version: 4.1.1 Package: apt-cacher Architecture: all diff -Nru apt-cacher-1.7.15/lib/apt-cacher.pl apt-cacher-1.7.16/lib/apt-cacher.pl --- apt-cacher-1.7.15/lib/apt-cacher.pl 2017-03-14 16:55:02.000000000 +0000 +++ apt-cacher-1.7.16/lib/apt-cacher.pl 2017-11-08 09:14:29.000000000 +0000 @@ -114,6 +114,8 @@ xenial yakkety zesty + artful + bionic )), user => $>, @@ -674,8 +676,9 @@ sub get_original_url { my ($filename) = @_; - # Infer from filename, assume HTTP - return URI->new('http://' . join('/', split(/_/, $filename))); + # Infer from filename, beware Translation-??_??, assume HTTP + $filename =~ s/(?new('http://' . $filename); } # Stores data flattened for use in tied hashes