diff -Nru popularity-contest-1.69ubuntu1/debian/changelog popularity-contest-1.70ubuntu1/debian/changelog --- popularity-contest-1.69ubuntu1/debian/changelog 2020-02-13 23:04:46.000000000 +0000 +++ popularity-contest-1.70ubuntu1/debian/changelog 2020-05-01 22:18:21.000000000 +0000 @@ -1,3 +1,44 @@ +popularity-contest (1.70ubuntu1) groovy; urgency=low + + * Merge from Debian unstable. Remaining changes: + - Set debconf question priority from "high" to "medium". + - Remove mail-transport-agent recommends. + - Branding changes in the description and homepage. + - Upload uncompressed reports and check for a server response of "Thanks". + - Set submiturls to popcon.ubuntu.com. + - Set email addresses to point to ubuntu.com. + - Don't encrypt reports; the Ubuntu server doesn't handle that. + - Drop gnupg Recommends to a Suggests to prevent bloating standard. + + -- Steve Langasek Fri, 01 May 2020 15:18:21 -0700 + +popularity-contest (1.70) unstable; urgency=low + + * debian-popcon.gpg: use new submission key + * debian/cron.daily: + - fix reporting logic to avoid double submissions. Closes: #930446 + - store last successful http submission timestamp in + /var/lib/popularity-contest/lastsub. + - run 'popularity-contest --su-nobody' as root. This allows + popcon to read the configuration file and /proc/*/maps files. + Closes: #865730. Thanks Robert Luberda. + - rename /var/log/popularity-contest.new.gpg to + /var/log/popularity-contest.gpg + * debian/control: + - Updated Standards-Version from 4.4.0 to 4.5.0. No change needed. + - Build-Depends on debhelper-compat (=12) + * debian/compat: removed + * Update example server-side scripts to popcon.d.o version: + - popanal.py: bump stable version to 1.67 + - popcon.pl: update URL from Alioth to Salsa + * popularity-contest: + - add private option --su-nobody to drop privileges after reading + the configuration file and /proc. + * examples/bin/popanal3.py: + - Python 3 version of popanal.py (experimental), will replace popanal.py + + -- Bill Allombert Mon, 30 Mar 2020 19:47:56 +0200 + popularity-contest (1.69ubuntu1) focal; urgency=low * Merge from Debian unstable. Remaining changes: diff -Nru popularity-contest-1.69ubuntu1/debian/compat popularity-contest-1.70ubuntu1/debian/compat --- popularity-contest-1.69ubuntu1/debian/compat 2019-07-18 15:45:22.000000000 +0000 +++ popularity-contest-1.70ubuntu1/debian/compat 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -9 diff -Nru popularity-contest-1.69ubuntu1/debian/control popularity-contest-1.70ubuntu1/debian/control --- popularity-contest-1.69ubuntu1/debian/control 2019-07-18 15:45:22.000000000 +0000 +++ popularity-contest-1.70ubuntu1/debian/control 2020-03-31 03:14:49.000000000 +0000 @@ -4,9 +4,9 @@ Maintainer: Ubuntu Developers XSBC-Original-Maintainer: Popularity Contest Developers Uploaders: Bill Allombert -Build-Depends: debhelper (>=9) +Build-Depends: debhelper-compat (= 12) Homepage: https://popcon.debian.org/ -Standards-Version: 4.4.0 +Standards-Version: 4.5.0 Vcs-Browser: https://salsa.debian.org/popularity-contest-team/popularity-contest Vcs-Git: https://salsa.debian.org/popularity-contest-team/popularity-contest.git diff -Nru popularity-contest-1.69ubuntu1/debian/cron.daily popularity-contest-1.70ubuntu1/debian/cron.daily --- popularity-contest-1.69ubuntu1/debian/cron.daily 2019-07-18 15:45:22.000000000 +0000 +++ popularity-contest-1.70ubuntu1/debian/cron.daily 2020-03-31 03:14:49.000000000 +0000 @@ -76,22 +76,37 @@ export http_proxy="$HTTP_PROXY"; fi +POPCONVAR=/var/lib/popularity-contest POPCONOLD=/var/log/popularity-contest POPCONNEW=/var/log/popularity-contest.new POPCON="$POPCONNEW" +last_sub() +{ + if [ -r "$POPCONVAR/lastsub" ] ; then + cat "$POPCONVAR/lastsub" + else + date -r "$POPCONOLD" +%s + fi +} +set_sub() +{ + test -d "$POPCONVAR" || mkdir -p "$POPCONVAR" + date +%s > "$POPCONVAR/lastsub" +} + # Only run on the given day, to spread the load on the server a bit -if [ "$DAY" ] && [ "$DAY" != "$(date +%w)" ] ; then +if [ "$MODE" != "--crond" ] || ( [ "$DAY" ] && [ "$DAY" != "$(date +%w)" ] ) ; then # Ensure that popcon runs at least once in the last week if [ -f "$POPCONOLD" ] ; then now=$(date +%s) - lastrun=$(date -r $POPCONOLD +%s) + lastrun=$(last_sub) if [ "$MODE" = "--crond" ]; then - # 6.5 days, in seconds - week=561600 - else # 7.5 days, in seconds week=648000 + else + # 6.5 days, in seconds + week=561600 fi if [ "$(( $now - $lastrun ))" -le "$week" ]; then exit 0 @@ -104,11 +119,6 @@ umask 022 savelog -c 7 popularity-contest >/dev/null -run_popcon() -{ - runuser -s /bin/sh -c "/usr/sbin/popularity-contest" nobody -} - do_sendmail() { if [ -n "$MAILFROM" ]; then @@ -120,7 +130,7 @@ # generate the popularity contest data -run_popcon > $POPCON +/usr/sbin/popularity-contest --su-nobody > $POPCON GPG=/usr/bin/gpg @@ -149,6 +159,7 @@ if setsid $TORIFY /usr/share/popularity-contest/popcon-upload \ -u $URL -f $POPCON -C 2>/dev/null ; then SUBMITTED=yes + set_sub else logger -t popularity-contest "unable to submit report to $URL." fi @@ -182,4 +193,7 @@ logger -t popularity-contest "unable to submit report." else mv $POPCONNEW $POPCONOLD + if [ -n "$POPCONGPG" ]; then + mv $POPCONNEW.gpg $POPCONOLD.gpg + fi fi Binary files /tmp/tmp7Hp9FM/74yCMIUF71/popularity-contest-1.69ubuntu1/debian-popcon.gpg and /tmp/tmp7Hp9FM/2DUWKx30FK/popularity-contest-1.70ubuntu1/debian-popcon.gpg differ diff -Nru popularity-contest-1.69ubuntu1/default.conf popularity-contest-1.70ubuntu1/default.conf --- popularity-contest-1.69ubuntu1/default.conf 2019-07-18 15:45:22.000000000 +0000 +++ popularity-contest-1.70ubuntu1/default.conf 2020-03-31 03:14:49.000000000 +0000 @@ -24,7 +24,7 @@ # popcon.debian.org. # KEYRING="/usr/share/popularity-contest/debian-popcon.gpg" -POPCONKEY="3F28C7EC54D00227C397BC7A1EA34EFBE7FD7002" +POPCONKEY="5B1A07804DD558242CF5538215A07BA5233E3E85" # MAILTO specifies the address to e-mail statistics to each week. # diff -Nru popularity-contest-1.69ubuntu1/examples/bin/popanal3.py popularity-contest-1.70ubuntu1/examples/bin/popanal3.py --- popularity-contest-1.69ubuntu1/examples/bin/popanal3.py 1970-01-01 00:00:00.000000000 +0000 +++ popularity-contest-1.70ubuntu1/examples/bin/popanal3.py 2020-03-30 17:47:56.000000000 +0000 @@ -0,0 +1,279 @@ +#!/usr/bin/python3 +# +# Read Debian popularity-contest submission data on stdin and produce +# some statistics about it. +# +import sys, string, time, glob, lzma + +mirrorbase = "/srv/mirrors/debian" +stable_version = "1.67" + +def ewrite(s): + sys.stderr.write("%s\n" % s) + +class Vote: + yes = 0 + old_unused = 0 + too_recent = 0 + empty_package = 0 + + def vote_for(vote, package, entry): + now = time.time() + if entry.atime == 0: # no atime: empty package + vote.empty_package = vote.empty_package + 1 + elif now - entry.atime > 30 * 24*3600: # 30 days since last use: old + vote.old_unused = vote.old_unused + 1 + elif now - entry.ctime < 30 * 24* 3600 \ + and entry.atime - entry.ctime < 24*3600: # upgraded too recently + vote.too_recent = vote.too_recent + 1 + else: # otherwise, vote for this package + vote.yes = vote.yes + 1 + +deplist = {} +provlist = {} + +class Stat: + def __init__(self): + self.vote = {} + self.vendor = {} + self.release = {} + self.arch = {} + self.count = 0 + + def output(self,filename): + out = open(filename, 'w') + out.write("Submissions: %8d\n" % self.count) + releaselist = list(self.release.keys()) + releaselist.sort() + for release in releaselist: + out.write("Release: %-30s %5d\n" + % (release, self.release[release])) + archlist = list(self.arch.keys()) + archlist.sort() + for arch in archlist: + out.write("Architecture: %-30s %5d\n" + % (arch, self.arch[arch])) + vendorlist = list(self.vendor.keys()) + vendorlist.sort() + for vendor in vendorlist: + out.write("Vendor: %-30s %5d\n" + % (vendor, self.vendor[vendor])) + pkglist = list(self.vote.keys()) + pkglist.sort() + for package in pkglist: + fv = self.vote[package] + out.write("Package: %-30s %5d %5d %5d %5d\n" + % (package, fv.yes, fv.old_unused, + fv.too_recent, fv.empty_package)) + out.close() + +stat = Stat() +stat_stable = Stat() + +def parse_depends(depline): + l = [] + split = depline.split(',') + for d in split: + x = d.split() + if (x): + l.append(x[0]) + return l + + +def read_depends(filename): + file = lzma.LZMAFile(filename, "r") + package = None + + while 1: + try: + line = str(file.readline(),encoding='latin_1') + except: + line = False + if line: + if line[0]==' ' or line[0]=='\t': continue # continuation + split = line.split(':') + + if not line or split[0]=='Package': + if package and (len(dep) > 0 or len(prov) > 0): + deplist[package] = dep + for d in prov: + if not (d in provlist): + provlist[d] = [] + provlist[d].append(package) + if package: + package = None + if line: + package = split[1].strip() + dep = [] + prov = [] + elif split[0]=='Depends' or split[0]=='Requires': + dep = dep + parse_depends(split[1]) + elif split[0]=='Provides': + prov = parse_depends(split[1]) + + if not line: break + +class Entry: + atime = 0; + ctime = 0; + mru_file = ''; + + def __init__(self, atime, ctime, mru_file): + try: + self.atime = int(atime) + self.ctime = int(ctime) + except: + self.atime = self.ctime = 0 + self.mru_file = mru_file + +def istimestamp(s): + return s.isdigit() or (s[0]=='-' and s[1:].isdigit()) + +class Submission: + # format: {package: [atime, ctime, mru_file]} + entries = {} + + start_date = 0 + + arch = "unknown" + release= "unknown" + vendor= "Debian" + + # initialize a new entry with known data + def __init__(self, version, owner_id, date): + self.entries = {} + self.start_date = int(date) + self.id = owner_id + + # process a line of input from the survey + def addinfo(self, split): + if (len(split) < 4 or not istimestamp(split[0]) + or not istimestamp(split[1])): + ewrite(self.id + ': Invalid input line: ' + repr(split)) + return + self.entries[split[2]] = Entry(split[0], split[1], split[3]) + + # update the atime of dependency to that of dependant, if newer + def update_atime(self, dependency, dependant): + if not (dependency in self.entries): return + e = self.entries[dependency] + f = self.entries[dependant] + if e.atime < f.atime: + e.atime = f.atime + e.ctime = f.ctime + + # we found the last line of the survey: finish it + def done(self, date, st): + st.count = st.count + 1 + for package in self.entries.keys(): + if package in deplist: + for d in deplist[package]: + self.update_atime(d, package) + if d in provlist: + for dd in provlist[d]: + self.update_atime(dd, package) + for package in self.entries.keys(): + if not (package in st.vote): + st.vote[package] = Vote() + st.vote[package].vote_for(package, self.entries[package]) + + if not (self.vendor in st.vendor): + st.vendor[self.vendor] = 1 + else: + st.vendor[self.vendor] = st.vendor[self.vendor] + 1 + + if not (self.release in st.release): + st.release[self.release] = 1 + else: + st.release[self.release] = st.release[self.release] + 1 + ewrite("#%s %s" % (st.release[self.release], self.release)) + + if not (self.arch in st.arch): + st.arch[self.arch] = 1 + else: + st.arch[self.arch] = st.arch[self.arch] + 1 + +def headersplit(pairs): + header = {} + for d in pairs: + list = d.split(':') + try: + key, value = list + header[key] = value + except: + pass + return header + + +def read_submissions(stream): + e = None + while 1: + line = str(stream.readline(),encoding='latin_1') + if not line: break + + split = line.split() + if not split: continue + + if split[0]=='POPULARITY-CONTEST-0': + header = headersplit(split[1:]) + + if not ('ID' in header) or not ('TIME' in header): + ewrite('Invalid header: ' + split[1]) + continue + + e = None + try: + e = Submission(0, header['ID'], header['TIME']) + except: + ewrite('Invalid date: ' + header['TIME'] + ' for ID ' + header['ID']) + continue + + if 'VENDOR' in header: + if header['VENDOR']=='': + e.vendor = 'unknown' + else: + e.vendor = header['VENDOR'] + + if 'POPCONVER' in header: + if header['POPCONVER']=='': + e.release = 'unknown' + else: + e.release = header['POPCONVER'] + + if 'ARCH' in header: + if header['ARCH']=='x86_64': + e.arch = 'amd64' + elif header['ARCH']=='i386-gnu': + e.arch = 'hurd-i386' + elif header['ARCH']=='': + e.arch = 'unknown' + else: + e.arch = header['ARCH'] + + elif split[0]=='END-POPULARITY-CONTEST-0' and e != None: + header = headersplit(split[1:]) + if 'TIME' in header: + try: + date = int(header['TIME']) + except: + ewrite('Invalid date: ' + header['TIME']) + continue + e.done(date,stat) + if e.release==stable_version: + e.done(date,stat_stable) + e = None + + elif e != None: + e.addinfo(split) + # end of while loop + + +# main program + +for d in glob.glob('%s/dists/stable/*/binary-i386/Packages.xz' % mirrorbase): + read_depends(d) +for d in glob.glob('%s/dists/unstable/*/binary-i386/Packages.xz' % mirrorbase): + read_depends(d) +read_submissions(sys.stdin.buffer) +stat.output("results3") +stat_stable.output("results3.stable") diff -Nru popularity-contest-1.69ubuntu1/examples/bin/popanal.py popularity-contest-1.70ubuntu1/examples/bin/popanal.py --- popularity-contest-1.69ubuntu1/examples/bin/popanal.py 2019-07-18 15:45:22.000000000 +0000 +++ popularity-contest-1.70ubuntu1/examples/bin/popanal.py 2020-03-31 03:14:49.000000000 +0000 @@ -6,7 +6,7 @@ import sys, string, time, glob, lzma mirrorbase = "/srv/mirrors/debian" -stable_version = "1.64" +stable_version = "1.67" def ewrite(s): sys.stderr.write("%s\n" % s) diff -Nru popularity-contest-1.69ubuntu1/examples/bin/popcon.pl popularity-contest-1.70ubuntu1/examples/bin/popcon.pl --- popularity-contest-1.69ubuntu1/examples/bin/popcon.pl 2019-07-18 15:45:22.000000000 +0000 +++ popularity-contest-1.70ubuntu1/examples/bin/popcon.pl 2020-03-31 03:14:49.000000000 +0000 @@ -218,7 +218,7 @@ print HTML < Made by Bill Allombert. Last generated on $date UTC.
- Popularity-contest project by Avery Pennarun, Bill Allombert and Petter Reinholdtsen. + Popularity-contest project by Avery Pennarun, Bill Allombert and Petter Reinholdtsen.
Debian theme by Stéphane Blondon, based on Debian template Copyright © 1997-2013 diff -Nru popularity-contest-1.69ubuntu1/popularity-contest popularity-contest-1.70ubuntu1/popularity-contest --- popularity-contest-1.69ubuntu1/popularity-contest 2019-07-18 15:45:22.000000000 +0000 +++ popularity-contest-1.70ubuntu1/popularity-contest 2020-03-31 03:14:49.000000000 +0000 @@ -53,6 +53,40 @@ exit 1; } +# List all mapped files +my %mapped; +if (opendir(PROC, "/proc")) +{ + my @procfiles = readdir(PROC); + closedir(PROC); + + foreach (@procfiles) + { + -d "/proc/$_" or next; + m{^[0-9]+$} or next; + + open MAPS, "/proc/$_/maps" or next; + while () + { + m{(/.*)} or next; + $mapped{$1} = 1; + } + close MAPS; + } +} + +if (defined($ARGV[0]) && $ARGV[0] eq "--su-nobody") +{ + my $user="nobody"; + my ($uid, $gid, $home, $shell) = (getpwnam($user))[2,3,7,8]; + $( = $) = $gid; + $< = $> = $uid; + $ENV{USER} = $user; + $ENV{LOGNAME} = $user; + $ENV{HOME} = $home; + $ENV{SHELL} = $shell; +} + # Architecture. my $debarch = `dpkg --print-architecture`; chomp $debarch; @@ -85,28 +119,6 @@ my %popcon=(); -# List all mapped files -my %mapped; -if (opendir(PROC, "/proc")) -{ - my @procfiles = readdir(PROC); - closedir(PROC); - - foreach (@procfiles) - { - -d "/proc/$_" or next; - m{^[0-9]+$} or next; - - open MAPS, "/proc/$_/maps" or next; - while () - { - m{(/.*)} or next; - $mapped{$1} = 1; - } - close MAPS; - } -} - # List files diverted by dpkg my %diverted; if (open DIVERSIONS, "env LC_ALL=C dpkg-divert --list|")