diff -Nru umegaya-0.13/apache2/error-modrewrite/index.html umegaya-1.0/apache2/error-modrewrite/index.html --- umegaya-0.13/apache2/error-modrewrite/index.html 1970-01-01 00:00:00.000000000 +0000 +++ umegaya-1.0/apache2/error-modrewrite/index.html 2014-06-04 12:20:07.000000000 +0000 @@ -0,0 +1,5 @@ +

Please enable Apache's rewrite module.

+ +

On Debian systems, this is done with the following command.

+ +

sudo a2enmod rewrite

diff -Nru umegaya-0.13/apache2/umegaya.conf umegaya-1.0/apache2/umegaya.conf --- umegaya-0.13/apache2/umegaya.conf 1970-01-01 00:00:00.000000000 +0000 +++ umegaya-1.0/apache2/umegaya.conf 2014-06-04 12:20:07.000000000 +0000 @@ -0,0 +1,23 @@ +Alias /umegaya/fields-stats /var/lib/umegaya/fields-stats +Alias /umegaya/license-stats /var/lib/umegaya/license-stats + + Alias /umegaya /usr/share/umegaya/htdocs/error-modrewrite + +Alias /umegaya /usr/share/umegaya/htdocs + + + Options +FollowSymLinks + + RewriteEngine on + RewriteRule ^$ /cgi-bin/umegaya?help=1 + RewriteRule ^table/([\w_\-]+)$ /cgi-bin/umegaya?table=1;key=$1 [B] + RewriteRule ^yaml/([\w_\-]+)$ /cgi-bin/umegaya?yaml=1;key=$1 [B] + RewriteRule ^([a-z0-9\.\-\+]+)/([\w_\-]+)$ /cgi-bin/umegaya?package=$1;key=$2 [B] + + + +# +# ServerName umegaya.example.com +# ServerAdmin umegaya@example.com +# DocumentRoot /usr/share/umegaya/htdocs +# diff -Nru umegaya-0.13/cgi-bin/umegaya umegaya-1.0/cgi-bin/umegaya --- umegaya-0.13/cgi-bin/umegaya 2013-04-13 09:28:56.000000000 +0000 +++ umegaya-1.0/cgi-bin/umegaya 2014-05-31 06:32:21.000000000 +0000 @@ -23,7 +23,7 @@ B is a gatherer of meta information about the software packaged in Debian. This metadata is accumlated by the package maintainer in the version control system containing the Debian souce package, in a file called -C, and it is collected by the Umegaya web service each time it +C, and it is collected by the Umegaya web service each time it is accessed. The Umegaya web service then aggregates information about all the packages it knows, and outputs as tables or YAML records that can be loaded in central information hubs like the Ultimate Debian Database. @@ -65,7 +65,7 @@ use BerkeleyDB; use CGI; -use LWP::Simple qw(get); +use Umegaya; use YAML::XS qw(Load Dump DumpFile); use YAML::AppConfig; @@ -128,7 +128,7 @@ exit; } -# The URL of the 'upstream' file is never refreshed. +# The URL of the package's repository is never refreshed. unless (defined ($stored{"$package:YAML-URL"})) { $stored{"$package:YAML-URL"} = qx(debcheckout -d $package | grep url | cut -f2) || "NA"; chomp $stored{"$package:YAML-URL"}; @@ -137,10 +137,10 @@ # Refrain to refresh if the last update is still fresh, to avoid loading Alioth. if (defined ($stored{"$package:YAML-REFRESH-DATE"})) { say "Not updated since:", time - $stored{"$package:YAML-REFRESH-DATE"} if $debug; - $stored{"$package:YAML-REFRESH-DATE"} = refresh_from_url($stored{"$package:YAML-URL"}) if time - $stored{"$package:YAML-REFRESH-DATE"} > $delay; + $stored{"$package:YAML-REFRESH-DATE"} = refresh($package) if time - $stored{"$package:YAML-REFRESH-DATE"} > $delay; } else { say "$package is not yet in the database, trying to pull its metadata…" if $debug; - $stored{"$package:YAML-REFRESH-DATE"} = refresh_from_url($stored{"$package:YAML-URL"}) + $stored{"$package:YAML-REFRESH-DATE"} = refresh($package) } # Output for http://localhost/umegaya/package/key URL. @@ -153,44 +153,59 @@ } say "No $key key for package ${package}." if $debug; -# Reconstructs an URL to the a target file in the Debian directory. -# For Git repositories on Alioth, it relies on the Gitweb interface. -sub guess_file_url { - my $file_url = shift; - my $file_to_download = shift; - if ($file_url =~ /^svn/) { - $file_url .= '/' unless $file_url =~ m(/$); - $file_url .= "debian/$file_to_download"; - } - $file_url =~ s|^git://git.debian.org/g?i?t?/?(.*)|http://git.debian.org/?p=$1;a=blob_plain;f=debian/$file_to_download;hb=HEAD| if $file_url =~ /^git/; - say "Will retreive $file_url for debian/$file_to_download." if $debug; - return $file_url; +sub packagePool ($) { + my $package = shift; + $package =~ /^(.)/; # first letter + return $1 +} + +sub getFile ($$) { + my $package_url = shift; + my $file = shift; + my $command = $package_url->download_command($file); + say "Running “$command”)" if $debug; + qx($command) +} + +sub storeFile ($$) { + my $file = shift; + my $fileContents = shift; + my $fullPath = $store_files . '/' . packagePool($package) . '/'. $file; + print "Writing $fullPath ... " if $debug; + open(my $fileHandle, ">", "$fullPath") + or die "Can not write $fullPath: $!"; + print $fileHandle $fileContents; + close($fileHandle); + say "done." if $debug; +} + +sub downloadFile ($$$) { + my $package_url = shift; + my $package = shift; + my $file = shift; + storeFile( $package . '.' . $file + , getFile ( $package_url, $file)); } # Refresh and return a time stamp; store files if asked. -sub refresh_from_url { - my $package_url = shift; - say "Refreshing data from $package_url" if $debug; - my $debian_upstream_url = guess_file_url($package_url, "upstream"); - my $package_metadata_yaml; - $package_metadata_yaml = svn_export($debian_upstream_url) if $debian_upstream_url =~ /^svn/; - $package_metadata_yaml = get($debian_upstream_url ) if $debian_upstream_url =~ /^http/; +sub refresh { + my $package = shift; + say "Refreshing data for $package" if $debug; + my $package_url = Umegaya->new( url => $stored{"$package:YAML-URL"}); + my $package_metadata_yaml = getFile($package_url, "upstream/metadata"); + if ( !$package_metadata_yaml ) { + $package_metadata_yaml = getFile($package_url, "upstream"); + } + if ( !$package_metadata_yaml ) { + $package_metadata_yaml = getFile($package_url, "upstream-metadata.yaml"); + } if ( $store_files ) { - $package =~ /^(.)/; - my $pool = $1; - open(my $debian_upstream, ">", "$store_files/$pool/${package}.upstream") - or die "Can not write $store_files/$pool/${package}.upstream: $!"; - print $debian_upstream $package_metadata_yaml; - close($debian_upstream); + storeFile($package . '.upstream', $package_metadata_yaml); if ( $debian_control ) { - my $debian_control_url = guess_file_url($package_url, "control"); - system(qq(svn cat "$debian_control_url" > $store_files/$pool/${package}.control)) if $debian_control_url =~ /^svn/; - system(qq(GET "$debian_control_url" > $store_files/$pool/${package}.control)) if $debian_control_url =~ /^http/; + downloadFile( $package_url, $package, 'control' ) } if ( $debian_copyright ) { - my $debian_copyright_url = guess_file_url($package_url, "copyright"); - system(qq(svn cat "$debian_copyright_url" > $store_files/$pool/${package}.copyright)) if $debian_copyright_url =~ /^svn/; - system(qq(GET "$debian_copyright_url" > $store_files/$pool/${package}.copyright)) if $debian_copyright_url =~ /^http/; + downloadFile( $package_url, $package, 'copyright' ) } } my $package_metadata; @@ -235,9 +250,3 @@ $stored{"${package}:YAML-ALL"} = $package_metadata_yaml; return time; } - -# For the moment only SVN is supported. -sub svn_export { - my $url = shift; - return qx(svn cat $url); -} diff -Nru umegaya-0.13/conf/apache.conf umegaya-1.0/conf/apache.conf --- umegaya-0.13/conf/apache.conf 2013-04-13 10:20:22.000000000 +0000 +++ umegaya-1.0/conf/apache.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,18 +0,0 @@ -Alias /umegaya/fields-stats /var/lib/umegaya/fields-stats -Alias /umegaya/license-stats /var/lib/umegaya/license-stats -Alias /umegaya /usr/share/umegaya/htdocs - - - Options +FollowSymLinks - RewriteEngine on - RewriteRule ^$ /cgi-bin/umegaya?help=1 - RewriteRule ^table/([\w_\-]+)$ /cgi-bin/umegaya?table=1;key=$1 [B] - RewriteRule ^yaml/([\w_\-]+)$ /cgi-bin/umegaya?yaml=1;key=$1 [B] - RewriteRule ^([a-z0-9\.\-\+]+)/([\w_\-]+)$ /cgi-bin/umegaya?package=$1;key=$2 [B] - - -# -# ServerName umegaya.example.com -# ServerAdmin umegaya@example.com -# DocumentRoot /usr/share/umegaya/htdocs -# diff -Nru umegaya-0.13/conf/umegaya.conf umegaya-1.0/conf/umegaya.conf --- umegaya-0.13/conf/umegaya.conf 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/conf/umegaya.conf 2014-05-28 22:15:57.000000000 +0000 @@ -29,7 +29,7 @@ # umegaya-ddc-ping sends messages to syslog if the following is uncommented. #messages: syslog -# Will store a copy of debian/upstream and debian/copyright if set to a location. +# Will store a copy of debian/upstream/metadata and debian/copyright if set to a location. #store_files: /var/lib/umegaya/packages-metadata # URLs for Umegaya @@ -38,4 +38,4 @@ ping_url: http://localhost/umegaya # Archive with upstream files for UDD bibref gatherer -#archive_for_bibref_gatherer: /var/lib/gforge/chroot/home/groups/blends/htdocs/packages-metadata/packages-metadata.tar.bz2 +#archive_for_bibref_gatherer: /srv/blends.debian.org/www/packages-metadata/packages-metadata.tar.bz2 diff -Nru umegaya-0.13/debian/apache2 umegaya-1.0/debian/apache2 --- umegaya-0.13/debian/apache2 1970-01-01 00:00:00.000000000 +0000 +++ umegaya-1.0/debian/apache2 2014-06-04 12:20:07.000000000 +0000 @@ -0,0 +1 @@ +conf apache2/umegaya.conf diff -Nru umegaya-0.13/debian/changelog umegaya-1.0/debian/changelog --- umegaya-0.13/debian/changelog 2013-04-13 10:27:38.000000000 +0000 +++ umegaya-1.0/debian/changelog 2014-06-04 12:35:45.000000000 +0000 @@ -1,3 +1,34 @@ +umegaya (1.0) unstable; urgency=low + + [ Charles Plessy ] + b4ca35b example-push-metadata: removed a space causing a syntax error. + ff5658c ddc-ping: further restrict to subjects starting with 'Accepted'. + 91ea80c Transition to Apache 2.4. + e3fdc6d ebcb7ec cea94ed Transfer in a separate module the functions for + converting URL to download commands. + 152defd Transfer umegaya-guess-url's logic to Umegaya.pm, and stop + reporting download URLs. + 9a5f01b Correct for downloading data from Blends managed Git repositories. + Closes: #731455 + e810177 Moved the source repository to collab-maint. + 78e50f2 Depend on curl instead of recommending it. + a48da63 Suggest cron-apt to keep debcheckout's results up to date. + 442e9f5 Override lintian warning ‘non-standard-dir-perm var/lib/umegaya/*’ + 04cb261 umegaya-adm: new functions to directly manipulate the Berkeley DB. + 3ef0bdf 09c861d f12824f umegaya-refresh-blends : correct bugs and bitrot. + 96b4b94 Conforms to Policy 3.9.5. + 2bfe95c Build-depend on apache2-dev instead of dh-apache2. + + [ Andreas Tille ] + f7cc82e Change default path to files for UDD bibref gatherer. + + [ Paul Wise ] + 728dadc Use debian/upstream/metadata, support others for backwards + compatibility. + 9dba911 Use curl instead of GET since it can fail on HTTP errors. + + -- Charles Plessy Wed, 04 Jun 2014 21:35:38 +0900 + umegaya (0.13) unstable; urgency=low 06dbcdc Hosting the source code on Branchable. diff -Nru umegaya-0.13/debian/control umegaya-1.0/debian/control --- umegaya-0.13/debian/control 2013-04-13 09:34:32.000000000 +0000 +++ umegaya-1.0/debian/control 2014-06-04 12:31:48.000000000 +0000 @@ -3,28 +3,32 @@ Section: database Priority: optional Build-Depends: debhelper (>= 9), + apache2-dev, pandoc, perl, perl-doc -Standards-Version: 3.9.4 -Vcs-Browser: http://source.umegaya.branchable.com/?p=source.git;a=tree -Vcs-Git: git://umegaya.branchable.com/ +Standards-Version: 3.9.5 +Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/umegaya.git +Vcs-Git: git://anonscm.debian.org/collab-maint/umegaya.git Homepage: http://umegaya.branchable.com Package: umegaya Architecture: all -Depends: devscripts, +Depends: curl, + devscripts, libberkeleydb-perl, + libmouse-perl, libyaml-libyaml-perl, libyaml-appconfig-perl, libwww-perl, subversion, ${perl:Depends}, ${misc:Depends} -Recommends: curl +Recommends: ${misc:Recommends} +Suggests: cron-apt Description: Umegaya is a MEtadata GAtherer using YAml Aggregator of meta information about the software packages. The central concept is that the metadata is accumlated by the package maintainer in a version control system containing the source package, in a file called - debian/upstream, and collected by the web aggregator each time it is accessed. - + debian/upstream/metadata, and collected by the web aggregator each time + it is accessed. diff -Nru umegaya-0.13/debian/copyright umegaya-1.0/debian/copyright --- umegaya-0.13/debian/copyright 2013-04-13 09:36:49.000000000 +0000 +++ umegaya-1.0/debian/copyright 2014-05-28 22:57:42.000000000 +0000 @@ -1,5 +1,5 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Source: http://source.umegaya.branchable.com/?p=source.git;a=tags +Source: git://anonscm.debian.org/collab-maint/umegaya.git Files: * Copyright: none claimed diff -Nru umegaya-0.13/debian/install umegaya-1.0/debian/install --- umegaya-0.13/debian/install 2013-04-13 10:28:08.000000000 +0000 +++ umegaya-1.0/debian/install 2014-06-04 12:20:07.000000000 +0000 @@ -1,5 +1,6 @@ +apache2/error-modrewrite/index.html usr/share/umegaya/htdocs/error-modrewrite +lib/Umegaya.pm usr/share/perl5 cgi-bin/umegaya usr/lib/cgi-bin -conf/apache.conf etc/umegaya conf/robots.txt etc/umegaya conf/umegaya.conf etc/umegaya scripts/umegaya-adm usr/bin diff -Nru umegaya-0.13/debian/links umegaya-1.0/debian/links --- umegaya-0.13/debian/links 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/debian/links 2014-06-04 12:20:07.000000000 +0000 @@ -1,2 +1 @@ /etc/umegaya/robots.txt /usr/share/umegaya/htdocs/robots.txt -/etc/umegaya/apache.conf /etc/apache2/conf.d/umegaya diff -Nru umegaya-0.13/debian/maintscript umegaya-1.0/debian/maintscript --- umegaya-0.13/debian/maintscript 1970-01-01 00:00:00.000000000 +0000 +++ umegaya-1.0/debian/maintscript 2014-06-04 12:20:07.000000000 +0000 @@ -0,0 +1 @@ +mv_conffile /etc/umegaya/apache.conf /etc/apache2/conf-available/umegaya.conf diff -Nru umegaya-0.13/debian/NEWS umegaya-1.0/debian/NEWS --- umegaya-0.13/debian/NEWS 1970-01-01 00:00:00.000000000 +0000 +++ umegaya-1.0/debian/NEWS 2014-06-04 12:32:32.000000000 +0000 @@ -0,0 +1,9 @@ +umegaya (1.0) unstable; urgency=low + + For the transition to apache 2.4, the file ‘/etc/umegaya/apache.conf’ was + moved to ‘/etc/apache2/conf-available/umegaya.conf’. + + umegaya-guess-url does not report guessed download URLs anymore. It guesses + the pakcage name from the URL. + + -- Charles Plessy Sat, 04 Jan 2014 22:21:10 +0900 diff -Nru umegaya-0.13/debian/postinst umegaya-1.0/debian/postinst --- umegaya-0.13/debian/postinst 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/debian/postinst 1970-01-01 00:00:00.000000000 +0000 @@ -1,47 +0,0 @@ -#!/bin/sh -# postinst script for umegaya -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -case "$1" in - configure) - if [ -x /etc/init.d/apache2 ] - then - if which /usr/sbin/invoke-rc.d >/dev/null 2>&1 - then - invoke-rc.d apache2 reload || true - else - /etc/init.d/apache2 reload || true - fi - fi - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff -Nru umegaya-0.13/debian/rules umegaya-1.0/debian/rules --- umegaya-0.13/debian/rules 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/debian/rules 2014-06-04 12:20:07.000000000 +0000 @@ -3,7 +3,7 @@ export DH_VERBOSE=1 %: - dh $@ + dh $@ --with apache2 override_dh_auto_build: perldoc -o nroff cgi-bin/umegaya > man/umegaya.1 diff -Nru umegaya-0.13/debian/umegaya.lintian-overrides umegaya-1.0/debian/umegaya.lintian-overrides --- umegaya-0.13/debian/umegaya.lintian-overrides 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/debian/umegaya.lintian-overrides 2014-05-31 03:14:14.000000000 +0000 @@ -1,2 +1,4 @@ # Umegaya is a recursive acronym. umegaya: description-starts-with-package-name +# The permissions are ‘drwxrwsr-x www-data/www-data’ to let apache write in. +umegaya: non-standard-dir-perm var/lib/umegaya/* diff -Nru umegaya-0.13/index.mdwn umegaya-1.0/index.mdwn --- umegaya-0.13/index.mdwn 2013-04-13 07:11:13.000000000 +0000 +++ umegaya-1.0/index.mdwn 2014-05-31 03:02:21.000000000 +0000 @@ -4,12 +4,13 @@ Umegaya is a gatherer of meta information about the software packaged in Debian. This metadata is accumlated by the package maintainer in the version control system containing the Debian souce package, in a file called -debian/upstream, and it is collected by the Umegaya web service -each time it is accessed. The Umegaya web service then aggregates information -about all the packages it knows, and outputs as tables or YAML records that can -be loaded in central information hubs like the Ultimate Debian Database. -Optionally, Umegaya will keep a local copy if the files it retrieves from the -VCS. See also [[!debwiki UpstreamMetadata]] on the Debian wiki. +[`debian/upstream/metadata`](http://dep.debian.net/deps/dep12), and it is +collected by the Umegaya web service each time it is accessed. The Umegaya web +service then aggregates information about all the packages it knows, and +outputs as tables or YAML records that can be loaded in central information +hubs like the [Ultimate Debian Database](https://udd.debian.org/). Optionally, +Umegaya will keep a local copy if the files it retrieves from the VCS. See +also [[!debwiki UpstreamMetadata]] on the Debian wiki. Umegaya is distributed under the BOLA [[LICENSE]]. diff -Nru umegaya-0.13/lib/Umegaya.pm umegaya-1.0/lib/Umegaya.pm --- umegaya-0.13/lib/Umegaya.pm 1970-01-01 00:00:00.000000000 +0000 +++ umegaya-1.0/lib/Umegaya.pm 2014-05-31 05:40:27.000000000 +0000 @@ -0,0 +1,60 @@ +package Umegaya; + +use Mouse; + +has 'url' => (is => 'rw', isa => 'Str'); + +sub download_command { + my $self = shift; + my $file = shift; + my $command = $self->url; # Starting with the URL, build a command. + # Use svn cat for packages hosted in Subversion. + if ($command =~ /^svn/) { + $command .= '/' unless $command =~ m(/$); + $command = "svn cat ${command}debian/$file"; + return $command; + } + # Download through gitweb for packages hosted on Alioth. + if ($command =~ m,^git://(?:git|anonscm).debian.org,) { + $command =~ s,^git://(?:git|anonscm).debian.org/g?i?t?/?(.*),http://anonscm.debian.org/gitweb/?p=$1;a=blob_plain;f=debian/$file;hb=HEAD,; + $command = qq(curl --fail --silent "$command"); + return $command; + } + # GitHub + if ($command =~ m,://github.com/,) { + $command =~ m,://github.com/([A-Za-z0-9\+\-\.]+)/([a-z0-9\+\-\.]+).git$,; + $command = "curl --fail --silent https://raw.github.com/$1/$2/master/debian/$file"; + return $command; + } + # Try git archive for other git-hosted packages. + if ($command =~ /^git/) { + $command = "git archive --remote=$command HEAD:debian $file | tar --extract --file - --to-stdout"; + return $command; + } + die "Could not guess command for " . $self->url; +} + +sub guess_package { + my $self = shift; + foreach my $guess ( + # Debian Med + "svn://svn.debian.org/debian-med/trunk/packages.*/([a-z0-9\+\-\.]+)/trunk", + # Debichem + "svn://svn.debian.org/s?v?n?/?debichem/unstable/([a-z0-9\+\-\.]+)/", + "svn://svn.debian.org/s?v?n?/?debichem/experimental/([a-z0-9\+\-\.]+)/", + "svn://svn.debian.org/s?v?n?/?debichem/wnpp/([a-z0-9\+\-\.]+)/", + # Debian Science + "svn://svn.debian.org/s?v?n?/?debian-science/packages/R/([a-z0-9\+\-\.]+)/", + "svn://svn.debian.org/s?v?n?/?debian-science/packages/([a-z0-9\+\-\.]+)/", + # Other repositories + "svn://svn.debian.org/.*/([a-z0-9\+\-\.]+)/trunk", + "git://git.debian.org/.+/([a-z0-9\+\-\.]+).git", + "git://anonscm.debian.org/.+/([a-z0-9\+\-\.]+).git", + "://github.com/[A-Za-z0-9\+\-\.]+/([a-z0-9\+\-\.]+).git" + ) { + return $1 if $self->url =~ m/$guess/; + } + return undef; +} + +1; diff -Nru umegaya-0.13/man/umegaya-guess-url.1.mdwn umegaya-1.0/man/umegaya-guess-url.1.mdwn --- umegaya-0.13/man/umegaya-guess-url.1.mdwn 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/man/umegaya-guess-url.1.mdwn 2014-01-04 09:45:21.000000000 +0000 @@ -1,10 +1,10 @@ % UMEGAYA-GUESS-URL(1) Umegaya User Manual % Charles Plessy -% April 8, 2012 +% January 4, 2014 # NAME -umegaya-guess-url - Converts a VCS URL in a URL to "debian/upstream" +umegaya-guess-url - Guesses a package name from a VCS URL. # SYNOPSIS @@ -14,23 +14,8 @@ VCS (version control system) URLs indicate where to clone or check out anonymously source packages. `umegaya-guess-url` attempts to guess the package -name and convert the VCS URL in either a Subversion URL that can be used by -`svn cat`, or in a HTTP URL that can be used to retrieve `debian/upstream` via -Gitweb. - -It recoginses VCS URLs from the following Debian Pure Blends. - - svn://svn.debian.org/debian-med/trunk/packages/+?/([a-z0-9\+\-\.]+)/trunk - git://git.debian.org/debian-med/([a-z0-9\+\-\.]+).git - - svn://svn.debian.org/s?v?n?/?debichem/unstable/([a-z0-9\+\-\.]+)/ - svn://svn.debian.org/s?v?n?/?debichem/experimental/([a-z0-9\+\-\.]+)/ - svn://svn.debian.org/s?v?n?/?debichem/wnpp/([a-z0-9\+\-\.]+)/ - git://git.debian.org/g?i?t?/?debichem/packages/([a-z0-9\+\-\.]+).git - - svn://svn.debian.org/s?v?n?/?debian-science/packages/R/([a-z0-9\+\-\.]+)/ - svn://svn.debian.org/s?v?n?/?debian-science/packages/([a-z0-9\+\-\.]+)/ - git://git.debian.org/g?i?t?/?debian-science/packages/([a-z0-9\+\-\.]+).git +name when only this URL is known. This is useful when harvesting VCS URLs from +the tasks files of Debian Pure Blends. # RETURNS @@ -38,8 +23,8 @@ # EXAMPLE - $ ./scripts/umegaya-guess-url $(debcheckout -d emboss | grep url | cut -f2) - emboss http://git.debian.org/?p=debian-med/emboss.git;a=blob_plain;f=debian/upstream;hb=HEAD + $ umegaya-guess-url $(debcheckout -d emboss | grep url | cut -f2) + emboss # SEE ALSO @@ -47,7 +32,7 @@ # SOURCE -http://git.debian.org/?p=users/plessy/umegaya.git +http://umegaya.branchable.com/ # LIMITATIONS diff -Nru umegaya-0.13/man.mdwn umegaya-1.0/man.mdwn --- umegaya-0.13/man.mdwn 2013-04-13 10:22:42.000000000 +0000 +++ umegaya-1.0/man.mdwn 2014-05-28 22:15:57.000000000 +0000 @@ -2,7 +2,7 @@ ====================== * [[umegaya-ddc-ping.1]] — Umegaya's debian-devel-changes pinger - * [[umegaya-guess-url.1]] — Converts a VCS URL in a URL to `debian/upstream` + * [[umegaya-guess-url.1]] — Converts a VCS URL in a URL to `debian/upstream/metadata` * [[umegaya-fields-stats.1]] — ranked list of fields from collected copyright files * [[umegaya-license-stats.1]] — ranked list of licenses from collected copyright files * [[umegaya-query.1]] — query a remote umegaya instance diff -Nru umegaya-0.13/scripts/example-push-metadata umegaya-1.0/scripts/example-push-metadata --- umegaya-0.13/scripts/example-push-metadata 2013-04-13 07:42:10.000000000 +0000 +++ umegaya-1.0/scripts/example-push-metadata 2013-04-27 04:09:08.000000000 +0000 @@ -14,7 +14,7 @@ POOL=/var/lib/umegaya/packages-metadata/ -STAMP= ${TMPDIR-/tmp}/umegaya-timestamp +STAMP=${TMPDIR-/tmp}/umegaya-timestamp rm -f $STAMP touch $STAMP diff -Nru umegaya-0.13/scripts/umegaya-adm umegaya-1.0/scripts/umegaya-adm --- umegaya-0.13/scripts/umegaya-adm 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/scripts/umegaya-adm 2014-06-01 09:29:11.000000000 +0000 @@ -66,6 +66,12 @@ In combination with B<--key> or B<--package>, delete the value(s) instead of displaying them. +=item B<--list-entries>, B<--show-entry>, B<--delete-entry> + +List all the entries, or show or delete one entry of the Umegaya database. +These entries are package names joined to a Umegaya key by a colon. Examples: +“umegaya:Name”, “umegaya:YAML-URL”. + =item B<-k>, B<--key> If set alone, C will display all the package-values pairs for the @@ -96,13 +102,16 @@ =cut -my ($delete, $package, $purge, $help, $key, $value, $verbose); +my ($delete, $deleteEntry, $listEntries, $package, $purge, $help, $key, $showEntry, $value, $verbose); -GetOptions ( 'delete' => \$delete , +GetOptions ( 'delete|d' => \$delete , + 'delete-entry=s'=> \$deleteEntry , + 'list-entries'=> \$listEntries , 'package|p=s' => \$package , 'purge' => \$purge , 'help' => \$help , 'key=s' => \$key , + 'show-entry=s'=> \$showEntry , 'value|v=s' => \$value , 'verbose' => \$verbose ); @@ -112,12 +121,30 @@ } if ($purge) { - die "$0: --purge can not be combined with other arguments.\n" if ($delete or $package or $key or $value); + die "$0: --purge can not be combined with other arguments.\n" if ($delete or $listEntries or $package or $key or $value); untie %stored; unlink $berkeleydb or die "$0: could not remove $berkeleydb: $!\n"; exit; } +if ($listEntries) { + die "$0: --list-entries can not be combined with other arguments.\n" if ($delete or $package or $purge or $key or $value); + say join "\n", sort keys (%stored); + exit; +} + +if ($showEntry) { + die "$0: --show-entries can not be combined with other arguments.\n" if ($delete or $package or $purge or $key or $value); + say $stored{$showEntry} // "Entry not found"; + exit; +} + +if ($deleteEntry) { + die "$0: --show-entries can not be combined with other arguments.\n" if ($delete or $package or $purge or $key or $value); + delete $stored{$deleteEntry}; + exit; +} + if ($value) { die "$0: --value option needs --package and --key to be set\n" unless ($package and $key); $stored{join(':', $package, $key)} = $value; @@ -181,6 +208,6 @@ =head1 SOURCE -http://anonscm.debian.org/gitweb/?p=users/plessy/umegaya.git;a=blob;f=scripts/umegaya-adm +http://anonscm.debian.org/gitweb/?p=collab-maint/umegaya.git;a=blob;f=scripts/umegaya-adm =cut diff -Nru umegaya-0.13/scripts/umegaya-ddc-ping umegaya-1.0/scripts/umegaya-ddc-ping --- umegaya-0.13/scripts/umegaya-ddc-ping 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/scripts/umegaya-ddc-ping 2013-04-27 05:14:35.000000000 +0000 @@ -5,7 +5,7 @@ # trigger a ping for the corresponding package in an Umegaya instance. URL=$(awk '/^ping_url:/ {print $2 ; nextfile}' /etc/umegaya/umegaya.conf) -PACKAGE=$(grep ^Subject -m1 | awk '{print $3}') +PACKAGE=$(grep ^Subject -m1 | awk '/Subject: Accepted/ {print $3}') MESSAGES=$(awk '/^messages:/ {print $2 ; nextfile}' /etc/umegaya/umegaya.conf) if [ "$PACKAGE" != '' ] ; then diff -Nru umegaya-0.13/scripts/umegaya-guess-url umegaya-1.0/scripts/umegaya-guess-url --- umegaya-0.13/scripts/umegaya-guess-url 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/scripts/umegaya-guess-url 2014-01-04 09:38:33.000000000 +0000 @@ -5,71 +5,18 @@ use 5.10.0; +use Umegaya; + my $package_url = shift; -my $package; -chomp $package_url; -given ($package_url) { - when (undef) { + +unless (defined($package_url)) { die "Usage: umegaya-guess-url Vcs-URL\n"; - } - when (/^svn/) { - # Debian Med - if (m|debian-med/trunk/packages|) { - if (m|svn://svn.debian.org/debian-med/trunk/packages.*/([a-z0-9\+\-\.]+)/trunk| ) { - $package = $1; - } else { - die <<"ERROR_MESSAGE" -Could not parse « $package_url ». -The package is located in Debian Med’s Subversion repository according to its -VCS URL, but it does not point at a trunk directory as it should. See the -following page for more information. -http://debian-med.alioth.debian.org/docs/policy.html#svn-repository-structure -ERROR_MESSAGE - } - } - # Debichem - elsif (m|svn://svn.debian.org/s?v?n?/?debichem/unstable/([a-z0-9\+\-\.]+)/|) { - $package = $1; - } - elsif (m|svn://svn.debian.org/s?v?n?/?debichem/experimental/([a-z0-9\+\-\.]+)/|) { - $package = $1; - } - elsif (m|svn://svn.debian.org/s?v?n?/?debichem/wnpp/([a-z0-9\+\-\.]+)/|) { - $package = $1; - } - # Debian Science - elsif (m|svn://svn.debian.org/s?v?n?/?debian-science/packages/R/([a-z0-9\+\-\.]+)/| ) { - $package = $1; - } - elsif (m|svn://svn.debian.org/s?v?n?/?debian-science/packages/([a-z0-9\+\-\.]+)/| ) { - $package = $1; - } - # Other repositories - elsif (m|svn://svn.debian.org/.*/([a-z0-9\+\-\.]+)/trunk| ) { - $package = $1; - } - else { - die "Could not detect package name for $package_url\n"; - } - $package_url .= "debian/upstream"; - } - when (/^git/) { - if (m|git://git.debian.org/.+/([a-z0-9\+\-\.]+).git|) { - $package = $1; - $package_url =~ s|^git://git.debian.org/g?i?t?/?(.*)|http://git.debian.org/?p=$1;a=blob_plain;f=debian/upstream;hb=HEAD|; - } - # GitHub (master branch only) - elsif (m|git://github.com/([A-Za-z0-9\+\-\.]+)/([a-z0-9\+\-\.]+).git$|) { - $package = $2; - $package_url = "https://raw.github.com/$1/$package/master/debian/upstream" - } - else { - die "Could not detect package name for $package_url\n"; - } - } - default { - die "Only Subversion and Git URLs are supported" - } } -say "$package\t$package_url"; +my $url = Umegaya->new(url => $package_url); + +unless (defined($url->guess_package)) { + die "Could not detect package name for $package_url\n"; +} + +say $url->guess_package; diff -Nru umegaya-0.13/scripts/umegaya-refresh-blends umegaya-1.0/scripts/umegaya-refresh-blends --- umegaya-0.13/scripts/umegaya-refresh-blends 2013-01-12 09:12:17.000000000 +0000 +++ umegaya-1.0/scripts/umegaya-refresh-blends 2014-06-02 23:03:35.000000000 +0000 @@ -1,10 +1,12 @@ -#!/bin/sh +#!/bin/bash set -e # Exit if refresh_blends is null in umegaya.conf. -[ $(grep ^refresh_blends /etc/umegaya/umegaya.conf | - sed -r 's/refresh_blends:\s+//') = 0 ] && exit 0 +[ ! "$(awk '/^refresh_blends/ {print $2}' /etc/umegaya/umegaya.conf)" = 1 ] && exit 0 + +# Debug mode ? +[ "$(awk '/^debug/ {print $2}' /etc/umegaya/umegaya.conf)" = 1 ] && DEBUG=1 # Exit if curl, debcheckout, umegaya-adm, umegaya-guess-url or svn are not installed. # Exit code is zero as this script is intended as a cron job. @@ -14,33 +16,45 @@ [ -x /usr/bin/umegaya-guess-url ] || exit 0 [ -x /usr/bin/svn ] || exit 0 -# Start with debichem med and science, but later let's fetch all the -# names from $(svn ls svn://svn.debian.org/blends/projects/) -BLENDS='debichem debian-med debian-science' +# Hardcode the location of the tasks files, since we are transitionning +# from Subversion to Git. -#set -x -for BLEND in $BLENDS -do - BLEND_URL=$(debcheckout -d $BLEND | grep url | cut -f2) +DEBICHEM_URL="svn://svn.debian.org/blends/projects/debichem/trunk/debichem" +DEBIAN_MED_URL="svn://svn.debian.org/blends/projects/med/trunk/debian-med" +DEBIAN_SCIENCE_URL="git://anonscm.debian.org/blends/projects/science.git" - # Learn VCS URL for packages not yet in Debian. - for TASK in $(svn ls $BLEND_URL/tasks) - do - for VCS_URL in $(svn cat $BLEND_URL/tasks/$TASK | grep 'Vcs-[SG]' | cut -f2 -d' ') +declare -A BLENDS + +BLENDS[debichem-URLs]='for file in $(svn ls $DEBICHEM_URL/tasks) ; do svn cat $DEBICHEM_URL/tasks/$file ; done' +BLENDS[debichem-pack]='svn cat $DEBICHEM_URL/debian/control' + +BLENDS[debian-med-URLs]='for file in $(svn ls $DEBIAN_MED_URL/tasks) ; do svn cat $DEBIAN_MED_URL/tasks/$file ; done' +BLENDS[debian-med-pack]='svn cat $DEBIAN_MED_URL/debian/control' + +BLENDS[debian-science-URLs]='git archive --remote=$DEBIAN_SCIENCE_URL HEAD:tasks | tar xf - --to-stdout' +BLENDS[debian-science-pack]='git archive --remote=$DEBIAN_SCIENCE_URL HEAD:debian control | tar xf - --to-stdout' + + +for BLEND in debichem debian-med debian-science +do +[ $DEBUG ] && echo "Next blend: $BLEND" +# Override the Umegaya database with the information from the tasks file. + for VCS_URL in $(eval ${BLENDS[$BLEND-URLs]} | awk '/^Vcs-[SG]/ {print $2}') do - PACKAGE=$(umegaya-guess-url $VCS_URL | cut -f1) - [ $PACKAGE ] && - umegaya-adm -p $PACKAGE -k YAML-URL -v $VCS_URL + [ $DEBUG ] && echo "$VCS_URL" + PACKAGE=$(umegaya-guess-url $VCS_URL | cut -f1) + [ $PACKAGE ] && + umegaya-adm -p $PACKAGE -k YAML-URL -v $VCS_URL done - done - - # Refresh Umegaya for each package. +# Refresh Umegaya for each package. PACKAGES=$( - svn cat $BLEND_URL/debian/control | - grep -e ^Recommends -e ^Suggests | - sed -e 's/,//g' -e 's/|//g' -e 's/Recommends://g' -e 's/Suggests://g' ) + eval ${BLENDS[$BLEND-pack]} | + grep-dctrl -s Recommends,Suggests - | + sed -e 's/,//g' -e 's/|//g' -e 's/Recommends://g' -e 's/Suggests://g' | + sort -u ) for PACKAGE in $PACKAGES do + [ $DEBUG ] && echo "$PACKAGE" SOURCE=$(apt-cache showsrc $PACKAGE 2> /dev/null | grep Package -m1 | awk '{print $2}') if [ "$SOURCE" != "" ] ; then # make sure a failed ping will not interupt script diff -Nru umegaya-0.13/setup.mdwn umegaya-1.0/setup.mdwn --- umegaya-0.13/setup.mdwn 2013-04-13 10:18:46.000000000 +0000 +++ umegaya-1.0/setup.mdwn 2014-05-31 03:15:15.000000000 +0000 @@ -6,26 +6,28 @@ - Apache - Berkeley DB + - curl - The following Perl modules: + - BerkeleyDB - LWP - - YAML + - Mouse + - YAML::XS - YAML::Appconfig -YAML::Appconfig is ageing and generates deprecation warnings with recent -versions of Perl, but this has been corrected in Debian's 0.16-2 package. - Configuration ------------- - - Set up an Apache virtual host using the `apache.conf` file in - `/etc/umegaya/`. - - Set up a directory where apache can write the `umegaya.db` Berkeley DB. By - default it is `/var/lib/umegaya`. + - Set up an Apache virtual host using the `umegaya.conf` file in + `/etc/apache2/conf-available/`. + - Set up a directory where Apache can write the `umegaya.db` Berkeley DB + and optionally save a copy of the upstream, copyright and control files + in the `packages-metatata` pool (by default in `/var/lib/umegaya`). - Make sure that the Umegaya administrator has also write access to the directory. - Umegaya finds the package's VCS using the `debcheckout` command of the devscripts package, so a `deb-src` line tracking Sid is necessary for a full - coverage. + coverage. The [[!debpkg cron-apt]] package is useful to keep the apt cache + up to date. Triggering ---------- @@ -56,12 +58,12 @@ This is true in theory, but in practice, a safeguard for errors may be necessary. -To determine if a package managed with Git on Alioth contains a debian/upstream file. +To determine if a package managed with Git on Alioth contains a debian/upstream/metadata file. for repo in /git/debian-med/*.git do (cd $repo ; git ls-tree master debian/ | - grep 'debian/upstream$' > /dev/null && echo "$repo") + grep 'debian/upstream/metadata$' > /dev/null && echo "$repo") done | sed -e 's|/git/debian-med/||' -e 's|.git$||' diff -Nru umegaya-0.13/todo.mdwn umegaya-1.0/todo.mdwn --- umegaya-0.13/todo.mdwn 2013-04-13 06:29:19.000000000 +0000 +++ umegaya-1.0/todo.mdwn 2014-05-28 22:15:57.000000000 +0000 @@ -1,8 +1,10 @@ To do ===== - - Problem: make sure that Apache as the write permission on the pool. This - will not be if a user creates a file with a usual umask. + - Failures to download the debian/upstream file should trigger a refractory + period using YAML-REFRESH-DATE so that umegaya-refresh-blends does not + hammer Alioth uselessly when a large number of binary packages correspond + to a single source package that has no debian/upstream file. - Keep an eye on https://joinup.ec.europa.eu/asset/adms_foss/description @@ -27,7 +29,7 @@ not package names. - Do not create empty file if contents were not retrieved. - Check if there is no buggy package names with spaces in the berkeley DB. - - Do not create 404 Gitweb files. + - Do not create 404 Gitweb files nor '500 read timeout' files. - See the example file 'example-push-metadata' on how to clean a package-metadata repository before pushing @@ -43,7 +45,7 @@ - dh_bibref ? - Regression tests. - Store YAML load errors. - - An example debian/upstream file, that illustrates YAML syntax. + - An example debian/upstream/metadata file, that illustrates YAML syntax. - Use a proper installation script instead of debian/install. - umegaya-adm: return package list when invoked with -k and -v @@ -62,12 +64,14 @@ - Integrate in the PTS. - - Detect all source packages that contain a debian/upstream file, for instance using - apt-file's "-a source" option (see "#632254"). + - Detect all source packages that contain a debian/upstream/metadata file, + for instance using apt-file's "-a source" option (see "#632254"). - Provide an index of all packages in the Umegaya database (or with upstream metadata in the UDD). - - Accept new debian/upstream files from the public. + - Accept new debian/upstream/metadata files from the public. - Integrate with http://duck.debian.net/ + + - Report better errors when a package does not contain upstream medatdata.