diff -Nru firefox-52.0+build2/browser/config/version_display.txt firefox-52.0.1+build2/browser/config/version_display.txt --- firefox-52.0+build2/browser/config/version_display.txt 2017-03-02 23:36:37.000000000 +0000 +++ firefox-52.0.1+build2/browser/config/version_display.txt 2017-03-17 13:41:07.000000000 +0000 @@ -1 +1 @@ -52.0 +52.0.1 diff -Nru firefox-52.0+build2/browser/config/version.txt firefox-52.0.1+build2/browser/config/version.txt --- firefox-52.0+build2/browser/config/version.txt 2017-03-02 23:36:37.000000000 +0000 +++ firefox-52.0.1+build2/browser/config/version.txt 2017-03-17 13:41:07.000000000 +0000 @@ -1 +1 @@ -52.0 +52.0.1 diff -Nru firefox-52.0+build2/config/milestone.txt firefox-52.0.1+build2/config/milestone.txt --- firefox-52.0+build2/config/milestone.txt 2017-03-02 23:36:38.000000000 +0000 +++ firefox-52.0.1+build2/config/milestone.txt 2017-03-17 13:41:08.000000000 +0000 @@ -10,4 +10,4 @@ # hardcoded milestones in the tree from these two files. #-------------------------------------------------------- -52.0 +52.0.1 diff -Nru firefox-52.0+build2/debian/changelog firefox-52.0.1+build2/debian/changelog --- firefox-52.0+build2/debian/changelog 2017-03-15 13:24:21.000000000 +0000 +++ firefox-52.0.1+build2/debian/changelog 2017-03-17 20:07:32.000000000 +0000 @@ -1,8 +1,7 @@ -firefox (52.0+build2-0ubuntu0.16.04.3) xenial-security; urgency=medium +firefox (52.0.1+build2-0ubuntu0.16.04.1) xenial-security; urgency=medium - * Fix LP: #1671079 - Don't crash if LOGNAME is not set in the environment - - add debian/patches/fix-lp1671079.patch - - update debian/patches/series + * New upstream stable release (52.0.1build2) + - Fix LP: #1671079 - Don't crash if LOGNAME is not set in the environment -- Chris Coulson Wed, 08 Mar 2017 15:44:11 +0000 diff -Nru firefox-52.0+build2/debian/patches/fix-lp1671079.patch firefox-52.0.1+build2/debian/patches/fix-lp1671079.patch --- firefox-52.0+build2/debian/patches/fix-lp1671079.patch 2017-03-15 13:24:21.000000000 +0000 +++ firefox-52.0.1+build2/debian/patches/fix-lp1671079.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -# HG changeset patch -# User Mike Hommey -# Date 1489468315 -32400 -# Node ID 51e13fb7f25488e25a8cbb834e46ec3f7bfd9a3e -# Parent d5fd5117ce4e2d0af055d5f8701a24b50754788c -Bug 1345413 - Fallback to getpwuid() info when LOGNAME is not set when creating the XRemote mutex. r=froydnj - -diff --git a/toolkit/xre/nsAppRunner.cpp b/toolkit/xre/nsAppRunner.cpp ---- a/toolkit/xre/nsAppRunner.cpp -+++ b/toolkit/xre/nsAppRunner.cpp -@@ -3815,21 +3815,37 @@ XREMain::XRE_mainStartup(bool* aExitFlag - const char* username = getenv("LOGNAME"); - const char* profile = nullptr; - - RemoteResult rr = ParseRemoteCommandLine(program, &profile, &username); - if (rr == REMOTE_ARG_BAD) { - return 1; - } - -+ if (!username) { -+ struct passwd *pw = getpwuid(geteuid()); -+ if (pw && pw->pw_name) { -+ // Beware that another call to getpwent/getpwname/getpwuid will overwrite -+ // pw, but we don't have such another call between here and when username -+ // is used last. -+ username = pw->pw_name; -+ } -+ } -+ - nsCOMPtr mutexDir; - rv = GetSpecialSystemDirectory(OS_TemporaryDirectory, getter_AddRefs(mutexDir)); - if (NS_SUCCEEDED(rv)) { -- nsAutoCString mutexPath = -- program + NS_LITERAL_CSTRING("_") + nsDependentCString(username); -+ nsAutoCString mutexPath = program + NS_LITERAL_CSTRING("_"); -+ // In the unlikely even that LOGNAME is not set and getpwuid failed, just -+ // don't put the username in the mutex directory. It will conflict with -+ // other users mutex, but the worst that can happen is that they wait for -+ // MOZ_XREMOTE_START_TIMEOUT_SEC during startup in that case. -+ if (username) { -+ mutexPath.Append(username); -+ } - if (profile) { - mutexPath.Append(NS_LITERAL_CSTRING("_") + nsDependentCString(profile)); - } - mutexDir->AppendNative(mutexPath); - - rv = mutexDir->Create(nsIFile::DIRECTORY_TYPE, 0700); - if (NS_SUCCEEDED(rv) || rv == NS_ERROR_FILE_ALREADY_EXISTS) { - mRemoteLockDir = mutexDir; diff -Nru firefox-52.0+build2/debian/patches/series firefox-52.0.1+build2/debian/patches/series --- firefox-52.0+build2/debian/patches/series 2017-03-15 13:24:21.000000000 +0000 +++ firefox-52.0.1+build2/debian/patches/series 2017-03-17 20:07:32.000000000 +0000 @@ -22,4 +22,3 @@ ubuntu-search-defaults.patch disable-geo-specific-search-defaults-for-us.patch revert-upstream-search-engine-changes.patch -fix-lp1671079.patch diff -Nru firefox-52.0+build2/dom/base/nsGlobalWindow.cpp firefox-52.0.1+build2/dom/base/nsGlobalWindow.cpp --- firefox-52.0+build2/dom/base/nsGlobalWindow.cpp 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/dom/base/nsGlobalWindow.cpp 2017-03-17 13:41:11.000000000 +0000 @@ -14998,6 +14998,10 @@ const Sequence& aLayout, ErrorResult& aRv) { + if (!ImageBitmap::ExtensionsEnabled(nullptr, nullptr)) { + aRv.Throw(NS_ERROR_TYPE_ERR); + return nullptr; + } if (aImage.IsArrayBuffer() || aImage.IsArrayBufferView()) { return ImageBitmap::Create(this, aImage, aOffset, aLength, aFormat, aLayout, aRv); diff -Nru firefox-52.0+build2/dom/workers/WorkerScope.cpp firefox-52.0.1+build2/dom/workers/WorkerScope.cpp --- firefox-52.0+build2/dom/workers/WorkerScope.cpp 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/dom/workers/WorkerScope.cpp 2017-03-17 13:41:11.000000000 +0000 @@ -476,6 +476,14 @@ const Sequence& aLayout, ErrorResult& aRv) { + JSContext* cx = GetCurrentThreadJSContext(); + MOZ_ASSERT(cx); + + if (!ImageBitmap::ExtensionsEnabled(cx, nullptr)) { + aRv.Throw(NS_ERROR_TYPE_ERR); + return nullptr; + } + if (aImage.IsArrayBuffer() || aImage.IsArrayBufferView()) { return ImageBitmap::Create(this, aImage, aOffset, aLength, aFormat, aLayout, aRv); diff -Nru firefox-52.0+build2/l10n/changesets firefox-52.0.1+build2/l10n/changesets --- firefox-52.0+build2/l10n/changesets 2017-03-02 23:47:16.000000000 +0000 +++ firefox-52.0.1+build2/l10n/changesets 2017-03-17 13:55:30.000000000 +0000 @@ -1,92 +1,92 @@ ach 778:34c236a22596 af 1014:90f7ccd90de4 -an 900:e7d65dba794e -ar 2335:ae3c3ed5a1ef -as 1112:ef4d8522e3d7 -ast 1885:b5b9db731849 -az 907:28b7ffc39eec -bg 1892:c0214fffc71e -bn-BD 1584:a177924d2d7e -bn-IN 1376:2e388c5db781 -br 1921:9468d5d2611f +an 903:6ac27fb55a82 +ar 2340:269c869ebd5a +as 1115:eda4bda0f52b +ast 1890:0c7e2e366772 +az 910:7aa8bf233833 +bg 1896:08f6918a2a89 +bn-BD 1588:290479c7a777 +bn-IN 1378:6bf15ca693ca +br 1925:315b4ca078c9 bs 920:afe1158833f4 -ca 3118:a7b9516b99cb -cak 636:bba0618bf2d2 -cs 4889:f8c4a8ac95ac -cy 1741:3427339b0214 -da 2854:9385081d2a31 -de 5628:95a9e7629b49 -dsb 942:ba808d110dcb -el 2016:4ed19ed728b7 -en-GB 2530:716d07ad7a25 -en-ZA 927:47d0d37e0e69 -eo 1437:6ccaf24fb19e -es-AR 3767:c0dd5c286899 -es-CL 1730:175e85d54c02 -es-ES 4948:1b7cea765c1c -es-MX 1646:818e4f18c036 -et 2323:9a62cc904fd6 -eu 2201:6664215afc8e -fa 1705:1f1f9c1f11fa -ff 1001:425b3ab91148 -fi 2885:a16e86eb47f4 -fr 7939:435fe2af5eb9 -fy-NL 3369:55b5dcd49e89 -ga-IE 2424:7ff1ea6d7043 -gd 2254:f8feae8925c2 -gl 3546:f9684fe14c45 -gn 315:a1f913d668ca -gu-IN 1592:e7280b4e086c -he 2065:a5f305538754 -hi-IN 1596:176e4496a493 -hr 2304:f998f3b9385a -hsb 1117:854c112f682f -hu 3199:47c217254ea5 -hy-AM 1808:42ee9c82060c -id 2279:f67d09ed89d1 -is 1958:d4bbac51ce98 -it 7547:d4af1c690181 -ja 3044:986b654cfbc1 -ja-JP-mac 2490:dfea8798b233 -ka 299:77b352c1d1b5 -kab 426:b52800111ecc -kk 1555:68aa69f8e074 +ca 3122:db0c8413d40d +cak 638:48976b536638 +cs 4893:885354f1cdbb +cy 1745:ba8b82a0fb06 +da 2858:e6e772bc765e +de 5632:dfc5dfd10294 +dsb 946:69df00f1b675 +el 2020:d9240cabedfc +en-GB 2534:7d52f46407eb +en-ZA 929:f347b59a8df1 +eo 1439:14423729e229 +es-AR 3772:7ba53b9f721b +es-CL 1732:04f51d907cb9 +es-ES 4952:d9aafadbba62 +es-MX 1648:800601315d8c +et 2327:f7860aabfcca +eu 2205:38bd3d22604c +fa 1707:c168661bcc79 +ff 1003:9a76c2de0e95 +fi 2889:0692b0a80c52 +fr 7943:d3b00c307306 +fy-NL 3373:d9e5b7d531f8 +ga-IE 2428:de3b060bf5a3 +gd 2258:7ddd900bbc35 +gl 3550:d4f02b30b6e3 +gn 317:c9940a67f207 +gu-IN 1594:f5e0b0f73ebf +he 2069:a4ccdb9d1f61 +hi-IN 1598:6fbf72ea6e8c +hr 2308:a29b9abb1582 +hsb 1121:a06937144633 +hu 3203:9f3e6f583aec +hy-AM 1812:60daaf9c13ec +id 2283:9b08600a6736 +is 1962:eee49dd7960a +it 7551:3ad2744cf7eb +ja 3048:d39a7712e219 +ja-JP-mac 2492:89dc0f7a454d +ka 301:31533d854abd +kab 429:4b480b6f87dd +kk 1557:32293f1352b0 km 1069:1d78cae948e2 -kn 1500:52138d354582 -ko 2591:a8cc6c5d1e22 +kn 1502:08358ea13142 +ko 2595:880ff2708612 lij 1005:221601dc2320 -lt 3710:fb44427e1eec -lv 1776:19c10b2cad0a -mai 1203:60a315a3fa89 +lt 3714:a0fc0fd6ac6a +lv 1778:a0a0cbc950bd +mai 1205:aee7ebacf181 mk 869:85557d511a18 -ml 1376:3479bc0b3456 -mr 1550:b5329e2176d6 -ms 1090:a3d7c90c5a43 -nb-NO 3329:e8dc0af29827 -nl 6238:4b43ab54ec05 -nn-NO 2060:0a21540d6ffb -or 1198:bc73714a9c80 -pa-IN 2331:92ba95891006 -pl 9658:9cbbf06ecb43 -pt-BR 3240:520a0e990e96 -pt-PT 5853:b48dad968f3c -rm 1866:cfe29f5cb78f -ro 2620:15036f8476ab -ru 4826:96a1c4277be8 -si 1602:a6e9f1cb4bb2 -sk 4650:bc63028b93c3 -sl 3114:26e5f011d7c1 -son 1269:1cf71b6e2f52 -sq 2494:c48b9692c652 -sr 1721:ac547f03d5e8 -sv-SE 5418:0f9636e9af97 -ta 1305:721e09baa82d -te 1572:b4103347c1de -th 1984:f5880a60817e -tr 2900:f5df48b36203 -uk 3794:bf2f0055d765 -uz 836:73e98e5ad667 -vi 1270:df556b077b9d -xh 1096:e6ad17aaeaae -zh-CN 3149:921183a45c0b -zh-TW 3242:b5c20cec93d1 +ml 1378:486cefd14c88 +mr 1552:07b512a80c8c +ms 1092:90d8affd0c9d +nb-NO 3333:f20add35140a +nl 6242:b0a1b28ce7c2 +nn-NO 2064:771acec62ede +or 1200:d347bb824e09 +pa-IN 2335:b04d94e0d546 +pl 9662:6c2e548b9bbe +pt-BR 3244:2b5aa934470a +pt-PT 5857:a77cca6a5b29 +rm 1870:5aab61500972 +ro 2624:83bf6b257b46 +ru 4830:79e0e415d559 +si 1604:69381f3cb9be +sk 4654:dc2d886d7a47 +sl 3118:3bc19f1d79f2 +son 1271:9f3f5c5826e9 +sq 2498:eca7094310bf +sr 1725:3180a53f9d21 +sv-SE 5422:ca10f580c369 +ta 1307:935f02027ee9 +te 1574:57c8ca3bcef9 +th 1986:0f9ce1d05b5b +tr 2904:a6bcafee41d4 +uk 3798:b2ed690c399b +uz 838:29e8c424eeb6 +vi 1272:9cdd4536afa9 +xh 1098:776a397631dc +zh-CN 3153:b414a70a6755 +zh-TW 3248:a374ce8540f9 diff -Nru firefox-52.0+build2/modules/libpref/init/all.js firefox-52.0.1+build2/modules/libpref/init/all.js --- firefox-52.0+build2/modules/libpref/init/all.js 2017-03-02 23:36:38.000000000 +0000 +++ firefox-52.0.1+build2/modules/libpref/init/all.js 2017-03-17 13:41:08.000000000 +0000 @@ -836,12 +836,8 @@ pref("canvas.path.enabled", true); pref("canvas.capturestream.enabled", true); -// Disable the ImageBitmap-extensions in the release build. -#ifdef RELEASE_OR_BETA +// Disable the ImageBitmap-extensions for now. pref("canvas.imagebitmap_extensions.enabled", false); -#else -pref("canvas.imagebitmap_extensions.enabled", true); -#endif // We want the ability to forcibly disable platform a11y, because // some non-a11y-related components attempt to bring it up. See bug diff -Nru firefox-52.0+build2/release/docker/beet-mover/Dockerfile firefox-52.0.1+build2/release/docker/beet-mover/Dockerfile --- firefox-52.0+build2/release/docker/beet-mover/Dockerfile 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/beet-mover/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ -FROM ubuntu:vivid - -RUN apt-get -q update \ - && apt-get install --yes -q \ - mercurial \ - python-dev \ - python-pip \ - python-virtualenv \ - libffi-dev \ - libssl-dev \ - libyaml-dev \ - libmysqlclient-dev \ - clamav \ - clamav-freshclam \ - curl \ - wget \ - && apt-get clean - -COPY requirements.txt /tmp/ -RUN pip install -r /tmp/requirements.txt -# Freshclam may be flaky, retry if it fails -RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done diff -Nru firefox-52.0+build2/release/docker/beet-mover/requirements.txt firefox-52.0.1+build2/release/docker/beet-mover/requirements.txt --- firefox-52.0+build2/release/docker/beet-mover/requirements.txt 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/beet-mover/requirements.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -sh -redo diff -Nru firefox-52.0+build2/release/docker/firefox-snap/distribution.ini firefox-52.0.1+build2/release/docker/firefox-snap/distribution.ini --- firefox-52.0+build2/release/docker/firefox-snap/distribution.ini 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/firefox-snap/distribution.ini 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ -[Global] -id=mozilla-snap -version=1.0 -about=Mozilla Firefox Snap - -[Preferences] -app.update.enabled=false -intl.locale.matchOS=true -browser.shell.checkDefaultBrowser=false diff -Nru firefox-52.0+build2/release/docker/firefox-snap/Dockerfile firefox-52.0.1+build2/release/docker/firefox-snap/Dockerfile --- firefox-52.0+build2/release/docker/firefox-snap/Dockerfile 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/firefox-snap/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update && apt-get install -qy snapcraft bzip2 curl && apt-get clean diff -Nru firefox-52.0+build2/release/docker/firefox-snap/Makefile firefox-52.0.1+build2/release/docker/firefox-snap/Makefile --- firefox-52.0+build2/release/docker/firefox-snap/Makefile 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/firefox-snap/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) -IMAGE_NAME = firefox-snapcraft -FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) - -build: - docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . - -push: - docker push $(FULL_IMAGE_NAME):latest - -pull: - docker pull $(FULL_IMAGE_NAME):latest diff -Nru firefox-52.0+build2/release/docker/firefox-snap/runme.sh firefox-52.0.1+build2/release/docker/firefox-snap/runme.sh --- firefox-52.0+build2/release/docker/firefox-snap/runme.sh 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/firefox-snap/runme.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,66 +0,0 @@ -#!/bin/sh - -set -xe - -# Required env variables -test $VERSION -test $BUILD_NUMBER -test $CANDIDATES_DIR - -# Optional env variables -: WORKSPACE ${WORKSPACE:=/home/worker/workspace} -: ARTIFACTS_DIR ${ARTIFACTS_DIR:=/home/worker/artifacts} - - -TARGET="firefox-${VERSION}.snap" - -mkdir -p "$ARTIFACTS_DIR" -rm -rf "${WORKSPACE}/source" && mkdir -p "${WORKSPACE}/source/opt" "${WORKSPACE}/source/usr/bin" - -CURL="curl --location --retry 10 --retry-delay 10" - -# Download and extract en-US linux64 binary -$CURL -o "${WORKSPACE}/firefox.tar.bz2" \ - "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/firefox-${VERSION}.tar.bz2" - -tar -C "${WORKSPACE}/source/opt" -xf "${WORKSPACE}/firefox.tar.bz2" -mkdir -p "${WORKSPACE}/source/opt/firefox/distribution/extensions" -cp -v distribution.ini "${WORKSPACE}/source/opt/firefox/distribution/" - -# Use release-specific list of locales to fetch L10N XPIs -$CURL -o "${WORKSPACE}/l10n_changesets.txt" "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/l10n_changesets.txt" -cat "${WORKSPACE}/l10n_changesets.txt" - -for locale in $(grep -v ja-JP-mac "${WORKSPACE}/l10n_changesets.txt" | awk '{print $1}'); do - $CURL -o "${WORKSPACE}/source/opt/firefox/distribution/extensions/langpack-${locale}@firefox.mozilla.org.xpi" \ - "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" -done - -# Symlink firefox binary to /usr/bin to make it available in PATH -ln -s ../../opt/firefox/firefox "${WORKSPACE}/source/usr/bin" - -# Generate snapcraft manifest -sed -e "s/@VERSION@/${VERSION}/g" -e "s/@BUILD_NUMBER@/${BUILD_NUMBER}/g" snapcraft.yaml.in > ${WORKSPACE}/snapcraft.yaml -cd ${WORKSPACE} -snapcraft - -mv *.snap "$ARTIFACTS_DIR/$TARGET" - -cd $ARTIFACTS_DIR - -# Generate checksums file -size=$(stat --printf="%s" $ARTIFACTS_DIR/$TARGET) -sha=$(sha512sum $ARTIFACTS_DIR/$TARGET | awk '{print $1}') -echo "$sha sha512 $size $TARGET" > $TARGET.checksums - -echo "Generating signing manifest" -hash=$(sha512sum $TARGET.checksums | awk '{print $1}') - -cat << EOF > signing_manifest.json -[{"file_to_sign": "$TARGET.checksums", "hash": "$hash"}] -EOF - -# For posterity -find . -ls -cat $TARGET.checksums -cat signing_manifest.json diff -Nru firefox-52.0+build2/release/docker/firefox-snap/snapcraft.yaml.in firefox-52.0.1+build2/release/docker/firefox-snap/snapcraft.yaml.in --- firefox-52.0+build2/release/docker/firefox-snap/snapcraft.yaml.in 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/firefox-snap/snapcraft.yaml.in 1970-01-01 00:00:00.000000000 +0000 @@ -1,37 +0,0 @@ -name: firefox -version: @VERSION@-@BUILD_NUMBER@ -summary: Mozilla Firefox web browser -description: Firefox is a powerful, extensible web browser with support for modern web application technologies. -confinement: strict - -apps: - firefox: - command: desktop-launch firefox - plugs: - - unity7 - - network - - home - - x11 - - opengl - - pulseaudio - - gsettings - - camera - - browser-sandbox - -plugs: - browser-sandbox: - interface: browser-support - allow-sandbox: true - -parts: - firefox: - plugin: dump - source: source - stage-packages: - - libxt6 - - libdbus-glib-1-2 - - libasound2 - - libpulse0 - - libgl1-mesa-dri - - libgl1-mesa-glx - after: [desktop-gtk3] diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/dep.pubkey firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/dep.pubkey --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/dep.pubkey 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/dep.pubkey 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzjHSobdeiQ3JHP/cCIOp -WaX9y12rL5mIo9OR9bpqEZdD0yXJJJeZA887Mv8slqsM+qObMUpKvfEE6zyYPIZJ -ANib31neI5BBYHhfhf2f5EnkilSYlmU3Gx+uRsmsdt58PpYe124tOAGgca/8bUy3 -eb6kUUTwvMI0oWQuPkGUaoHVQyj/bBMTrIkyF3UbfFtiX/SfOPvIoabNUe+pQHUe -pqC2+RxzDGj+shTq/hYhtXlptFzsEEb2+0foLy0MY8C30dP2QqbM2iavvr/P8OcS -Gm3H0TQcRzIEBzvPcIjiZi1nQj/r/3TlYRNCjuYT/HsNLXrB/U5Tc990jjAUJxdH -0wIDAQAB ------END PUBLIC KEY----- diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/Dockerfile firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/Dockerfile --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/Dockerfile 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,33 +0,0 @@ -FROM ubuntu:vivid -MAINTAINER Rail Aliiev - -# Required software -ENV DEBIAN_FRONTEND noninteractive -# Chain apt-get commands with apt-get clean in a single docker RUN -# to make sure that files are removed within a single docker layer -RUN apt-get update -q && \ - apt-get install -yyq --no-install-recommends \ - python mercurial curl python-boto python-setuptools python-cryptography && \ - apt-get clean - -COPY requirements.txt /tmp/ -# python-pip installs a lot of dependencies increasing the size of an image -# drastically. -RUN easy_install pip -RUN pip install -r /tmp/requirements.txt - -RUN hg clone https://hg.mozilla.org/build/tools /home/worker/tools - -RUN useradd -d /home/worker -s /bin/bash -m worker - -RUN mkdir /home/worker/bin -COPY scripts/* /home/worker/bin/ -RUN mkdir /home/worker/keys -COPY *.pubkey /home/worker/keys/ -COPY runme.sh /runme.sh -RUN chmod 755 /home/worker/bin/* /runme.sh - -ENV HOME /home/worker -ENV SHELL /bin/bash -ENV USER worker -ENV LOGNAME worker diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/Makefile firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/Makefile --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/Makefile 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) -IMAGE_NAME = funsize-balrog-submitter -FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) - -build: - docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . - -push: - docker push $(FULL_IMAGE_NAME):latest - -pull: - docker pull $(FULL_IMAGE_NAME):latest - -update_pubkeys: - curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey - curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey - curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/nightly.pubkey firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/nightly.pubkey --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/nightly.pubkey 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/nightly.pubkey 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4j/IS3gWbyVVnWn4ZRnC -Fuzb6VAaHa0I+4E504ekhVAhbKlSfBstkLbXajdjUVAJpn02zWnOaTl5KAdpDpIp -SkdA4mK20ej3/Ij7gIt8IwaX+ArXL8mP84pxDn5BgaNADm3206Z6YQzc/TDYu529 -qkDFmLqNUVRJAhPO+qqhKHIcVGh8HUHXN6XV1qOFip+UU0M474jAGgurVmAv8Rh7 -VvM0v5KmB6V6WHwM5gwjg2yRY/o+xYIsNeSes9rpp+MOs/RnUA6LI4WZGY4YahvX -VclIXBDgbWPYtojexIJkmYj8JIIRsh3eCsrRRe14fq7cBurp3CxBYMlDHf0RUoaq -hQIDAQAB ------END PUBLIC KEY----- diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/release.pubkey firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/release.pubkey --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/release.pubkey 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/release.pubkey 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvH4r94FpQ0gvr1hhTfV9 -NUeWPJ5CN6TZRq7v/Dc4nkJ1J4IP1B3UEii34tcNKpy1nKupiZuTT6T1zQYT+z5x -3UkDF9qQboQ8RNb/BEz/cN3on/LTEnZ7YSraRL11M6cEB8mvmJxddCEquwqccRbs -Usp8WUB7uRv1w6Anley7N9F/LE1iLPwJasZypRnzWb3aYsJy0cMFOYy+OXVdpktn -qYqlNIjnt84u4Nil6UXnBbIJNUVOCY8wOFClNvVpubjPkWK1gtdWy3x/hJU5RpAO -K9cnHxq4M/I4SUWTWO3r7yweQiHG4Jyoc7sP1jkwjBkSG93sDEycfwOdOoZft3wN -sQIDAQAB ------END PUBLIC KEY----- diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/requirements.txt firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/requirements.txt --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/requirements.txt 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/requirements.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -mar==1.2 diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/runme.sh firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/runme.sh --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/runme.sh 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/runme.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ -#!/bin/bash - -set -xe - -test $PARENT_TASK_ARTIFACTS_URL_PREFIX -test $BALROG_API_ROOT -test $SIGNING_CERT - -ARTIFACTS_DIR="/home/worker/artifacts" -mkdir -p "$ARTIFACTS_DIR" - -curl --location --retry 10 --retry-delay 10 -o "$ARTIFACTS_DIR/manifest.json" \ - "$PARENT_TASK_ARTIFACTS_URL_PREFIX/manifest.json" - -cat "$ARTIFACTS_DIR/manifest.json" -python /home/worker/bin/funsize-balrog-submitter.py \ - --artifacts-url-prefix "$PARENT_TASK_ARTIFACTS_URL_PREFIX" \ - --manifest "$ARTIFACTS_DIR/manifest.json" \ - -a "$BALROG_API_ROOT" \ - --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ - --verbose \ - $EXTRA_BALROG_SUBMITTER_PARAMS diff -Nru firefox-52.0+build2/release/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py --- firefox-52.0+build2/release/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,207 +0,0 @@ -#!/usr/bin/env python -import site -import os -import logging -import argparse -import json -import sys -import hashlib -import requests -import tempfile -from boto.s3.connection import S3Connection -from mardor.marfile import MarFile - -site.addsitedir("/home/worker/tools/lib/python") - -from balrog.submitter.cli import NightlySubmitterV4, ReleaseSubmitterV4 -from util.retry import retry, retriable - -log = logging.getLogger(__name__) - - -def get_hash(content, hash_type="md5"): - h = hashlib.new(hash_type) - h.update(content) - return h.hexdigest() - - -@retriable() -def download(url, dest, mode=None): - log.debug("Downloading %s to %s", url, dest) - r = requests.get(url) - r.raise_for_status() - - bytes_downloaded = 0 - with open(dest, 'wb') as fd: - for chunk in r.iter_content(4096): - fd.write(chunk) - bytes_downloaded += len(chunk) - - log.debug('Downloaded %s bytes', bytes_downloaded) - if 'content-length' in r.headers: - log.debug('Content-Length: %s bytes', r.headers['content-length']) - if bytes_downloaded != int(r.headers['content-length']): - raise IOError('Unexpected number of bytes downloaded') - - if mode: - log.debug("chmod %o %s", mode, dest) - os.chmod(dest, mode) - - -def verify_signature(mar, signature): - log.info("Checking %s signature", mar) - m = MarFile(mar, signature_versions=[(1, signature)]) - m.verify_signatures() - - -def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key, - mar_url, mar_dest, signing_cert): - conn = S3Connection(aws_access_key_id, aws_secret_access_key) - bucket = conn.get_bucket(bucket_name) - _, dest = tempfile.mkstemp() - log.info("Downloading %s to %s...", mar_url, dest) - download(mar_url, dest) - log.info("Verifying the signature...") - if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"): - verify_signature(dest, signing_cert) - for name in possible_names(mar_dest, 10): - log.info("Checking if %s already exists", name) - key = bucket.get_key(name) - if not key: - log.info("Uploading to %s...", name) - key = bucket.new_key(name) - # There is a chance for race condition here. To avoid it we check - # the return value with replace=False. It should be not None. - length = key.set_contents_from_filename(dest, replace=False) - if length is None: - log.warn("Name race condition using %s, trying again...", name) - continue - else: - # key.make_public() may lead to race conditions, because - # it doesn't pass version_id, so it may not set permissions - bucket.set_canned_acl(acl_str='public-read', key_name=name, - version_id=key.version_id) - # Use explicit version_id to avoid using "latest" version - return key.generate_url(expires_in=0, query_auth=False, - version_id=key.version_id) - else: - if get_hash(key.get_contents_as_string()) == \ - get_hash(open(dest).read()): - log.info("%s has the same MD5 checksum, not uploading...", - name) - return key.generate_url(expires_in=0, query_auth=False, - version_id=key.version_id) - log.info("%s already exists with different checksum, " - "trying another one...", name) - - raise RuntimeError("Cannot generate a unique name for %s", mar_dest) - - -def possible_names(initial_name, amount): - """Generate names appending counter before extension""" - prefix, ext = os.path.splitext(initial_name) - return [initial_name] + ["{}-{}{}".format(prefix, n, ext) for n in - range(1, amount + 1)] - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--artifacts-url-prefix", required=True, - help="URL prefix for MAR") - parser.add_argument("--manifest", required=True) - parser.add_argument("-a", "--api-root", required=True, - help="Balrog API root") - parser.add_argument("-d", "--dummy", action="store_true", - help="Add '-dummy' suffix to branch name") - parser.add_argument("--signing-cert", required=True) - parser.add_argument("-v", "--verbose", action="store_const", - dest="loglevel", const=logging.DEBUG, - default=logging.INFO) - args = parser.parse_args() - logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", - level=args.loglevel) - logging.getLogger("requests").setLevel(logging.WARNING) - logging.getLogger("boto").setLevel(logging.WARNING) - - balrog_username = os.environ.get("BALROG_USERNAME") - balrog_password = os.environ.get("BALROG_PASSWORD") - if not balrog_username and not balrog_password: - raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment " - "variables should be set") - - s3_bucket = os.environ.get("S3_BUCKET") - aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID") - aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY") - if not (s3_bucket and aws_access_key_id and aws_secret_access_key): - log.warn("Skipping S3 uploads...") - uploads_enabled = False - else: - uploads_enabled = True - - manifest = json.load(open(args.manifest)) - auth = (balrog_username, balrog_password) - - for e in manifest: - complete_info = [{ - "hash": e["to_hash"], - "size": e["to_size"], - }] - partial_info = [{ - "hash": e["hash"], - "size": e["size"], - }] - - if "previousVersion" in e and "previousBuildNumber" in e: - log.info("Release style balrog submission") - partial_info[0]["previousVersion"] = e["previousVersion"] - partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"] - submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth, - dummy=args.dummy) - retry(lambda: submitter.run( - platform=e["platform"], productName=e["appName"], - version=e["toVersion"], - build_number=e["toBuildNumber"], - appVersion=e["version"], extVersion=e["version"], - buildID=e["to_buildid"], locale=e["locale"], - hashFunction='sha512', - partialInfo=partial_info, completeInfo=complete_info, - )) - elif "from_buildid" in e and uploads_enabled: - log.info("Nightly style balrog submission") - partial_mar_url = "{}/{}".format(args.artifacts_url_prefix, - e["mar"]) - complete_mar_url = e["to_mar"] - dest_prefix = "{branch}/{buildid}".format( - branch=e["branch"], buildid=e["to_buildid"]) - partial_mar_dest = "{}/{}".format(dest_prefix, e["mar"]) - complete_mar_filename = "{appName}-{branch}-{version}-" \ - "{platform}-{locale}.complete.mar" - complete_mar_filename = complete_mar_filename.format( - appName=e["appName"], branch=e["branch"], - version=e["version"], platform=e["platform"], - locale=e["locale"] - ) - complete_mar_dest = "{}/{}".format(dest_prefix, - complete_mar_filename) - partial_info[0]["url"] = verify_copy_to_s3( - s3_bucket, aws_access_key_id, aws_secret_access_key, - partial_mar_url, partial_mar_dest, args.signing_cert) - complete_info[0]["url"] = verify_copy_to_s3( - s3_bucket, aws_access_key_id, aws_secret_access_key, - complete_mar_url, complete_mar_dest, args.signing_cert) - partial_info[0]["from_buildid"] = e["from_buildid"] - submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth, - dummy=args.dummy) - retry(lambda: submitter.run( - platform=e["platform"], buildID=e["to_buildid"], - productName=e["appName"], branch=e["branch"], - appVersion=e["version"], locale=e["locale"], - hashFunction='sha512', extVersion=e["version"], - partialInfo=partial_info, completeInfo=complete_info), - attempts=30, sleeptime=10, max_sleeptime=60, - ) - else: - raise RuntimeError("Cannot determine Balrog submission style") - -if __name__ == '__main__': - main() diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/dep.pubkey firefox-52.0.1+build2/release/docker/funsize-update-generator/dep.pubkey --- firefox-52.0+build2/release/docker/funsize-update-generator/dep.pubkey 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/dep.pubkey 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzjHSobdeiQ3JHP/cCIOp -WaX9y12rL5mIo9OR9bpqEZdD0yXJJJeZA887Mv8slqsM+qObMUpKvfEE6zyYPIZJ -ANib31neI5BBYHhfhf2f5EnkilSYlmU3Gx+uRsmsdt58PpYe124tOAGgca/8bUy3 -eb6kUUTwvMI0oWQuPkGUaoHVQyj/bBMTrIkyF3UbfFtiX/SfOPvIoabNUe+pQHUe -pqC2+RxzDGj+shTq/hYhtXlptFzsEEb2+0foLy0MY8C30dP2QqbM2iavvr/P8OcS -Gm3H0TQcRzIEBzvPcIjiZi1nQj/r/3TlYRNCjuYT/HsNLXrB/U5Tc990jjAUJxdH -0wIDAQAB ------END PUBLIC KEY----- diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/Dockerfile firefox-52.0.1+build2/release/docker/funsize-update-generator/Dockerfile --- firefox-52.0+build2/release/docker/funsize-update-generator/Dockerfile 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,33 +0,0 @@ -FROM ubuntu:vivid -MAINTAINER Rail Aliiev - -# Required software -ENV DEBIAN_FRONTEND noninteractive -# Chain apt-get commands with apt-get clean in a single docker RUN -# to make sure that files are removed within a single docker layer -RUN apt-get update -q && \ - apt-get install -yyq --no-install-recommends \ - python python-setuptools python-cryptography libgetopt-simple-perl \ - bzip2 clamav clamav-freshclam python-requests python-sh curl && \ - apt-get clean -RUN useradd -d /home/worker -s /bin/bash -m worker -COPY requirements.txt /tmp/ -# python-pip installs a lot of dependencies increasing the size of an image -# drastically. Using easy_install saves us almost 200M. -RUN easy_install pip -RUN pip install -r /tmp/requirements.txt - -# scripts -RUN mkdir /home/worker/bin -COPY scripts/* /home/worker/bin/ -COPY runme.sh /runme.sh -RUN chmod 755 /home/worker/bin/* /runme.sh -RUN mkdir /home/worker/keys -COPY *.pubkey /home/worker/keys/ -# Freshclam may be flaky, retry if it fails -RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done - -ENV HOME /home/worker -ENV SHELL /bin/bash -ENV USER worker -ENV LOGNAME worker diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/Makefile firefox-52.0.1+build2/release/docker/funsize-update-generator/Makefile --- firefox-52.0+build2/release/docker/funsize-update-generator/Makefile 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) -IMAGE_NAME = funsize-update-generator -FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) - -build: - docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . - -push: - docker push $(FULL_IMAGE_NAME):latest - -pull: - docker pull $(FULL_IMAGE_NAME):latest - -update_pubkeys: - curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey - curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey - curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/nightly.pubkey firefox-52.0.1+build2/release/docker/funsize-update-generator/nightly.pubkey --- firefox-52.0+build2/release/docker/funsize-update-generator/nightly.pubkey 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/nightly.pubkey 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4j/IS3gWbyVVnWn4ZRnC -Fuzb6VAaHa0I+4E504ekhVAhbKlSfBstkLbXajdjUVAJpn02zWnOaTl5KAdpDpIp -SkdA4mK20ej3/Ij7gIt8IwaX+ArXL8mP84pxDn5BgaNADm3206Z6YQzc/TDYu529 -qkDFmLqNUVRJAhPO+qqhKHIcVGh8HUHXN6XV1qOFip+UU0M474jAGgurVmAv8Rh7 -VvM0v5KmB6V6WHwM5gwjg2yRY/o+xYIsNeSes9rpp+MOs/RnUA6LI4WZGY4YahvX -VclIXBDgbWPYtojexIJkmYj8JIIRsh3eCsrRRe14fq7cBurp3CxBYMlDHf0RUoaq -hQIDAQAB ------END PUBLIC KEY----- diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/release.pubkey firefox-52.0.1+build2/release/docker/funsize-update-generator/release.pubkey --- firefox-52.0+build2/release/docker/funsize-update-generator/release.pubkey 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/release.pubkey 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvH4r94FpQ0gvr1hhTfV9 -NUeWPJ5CN6TZRq7v/Dc4nkJ1J4IP1B3UEii34tcNKpy1nKupiZuTT6T1zQYT+z5x -3UkDF9qQboQ8RNb/BEz/cN3on/LTEnZ7YSraRL11M6cEB8mvmJxddCEquwqccRbs -Usp8WUB7uRv1w6Anley7N9F/LE1iLPwJasZypRnzWb3aYsJy0cMFOYy+OXVdpktn -qYqlNIjnt84u4Nil6UXnBbIJNUVOCY8wOFClNvVpubjPkWK1gtdWy3x/hJU5RpAO -K9cnHxq4M/I4SUWTWO3r7yweQiHG4Jyoc7sP1jkwjBkSG93sDEycfwOdOoZft3wN -sQIDAQAB ------END PUBLIC KEY----- diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/requirements.txt firefox-52.0.1+build2/release/docker/funsize-update-generator/requirements.txt --- firefox-52.0+build2/release/docker/funsize-update-generator/requirements.txt 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/requirements.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -mar==1.2 -redo diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/runme.sh firefox-52.0.1+build2/release/docker/funsize-update-generator/runme.sh --- firefox-52.0+build2/release/docker/funsize-update-generator/runme.sh 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/runme.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,25 +0,0 @@ -#!/bin/sh - -set -xe - -test $TASK_ID -test $SIGNING_CERT - -ARTIFACTS_DIR="/home/worker/artifacts" -mkdir -p "$ARTIFACTS_DIR" - -curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \ - "https://queue.taskcluster.net/v1/task/$TASK_ID" - -# enable locale cache -export MBSDIFF_HOOK="/home/worker/bin/mbsdiff_hook.sh -c /tmp/fs-cache" - -if [ ! -z $FILENAME_TEMPLATE ]; then - EXTRA_PARAMS="--filename-template $FILENAME_TEMPLATE $EXTRA_PARAMS" -fi - -/home/worker/bin/funsize.py \ - --artifacts-dir "$ARTIFACTS_DIR" \ - --task-definition /home/worker/task.json \ - --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ - $EXTRA_PARAMS diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/scripts/funsize.py firefox-52.0.1+build2/release/docker/funsize-update-generator/scripts/funsize.py --- firefox-52.0+build2/release/docker/funsize-update-generator/scripts/funsize.py 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/scripts/funsize.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,275 +0,0 @@ -#!/usr/bin/env python - -import ConfigParser -import argparse -import functools -import hashlib -import json -import logging -import os -import shutil -import tempfile -import requests -import sh - -import redo -from mardor.marfile import MarFile - -log = logging.getLogger(__name__) -ALLOWED_URL_PREFIXES = [ - "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/", - "http://download.cdn.mozilla.net/pub/firefox/nightly/", - "https://mozilla-nightly-updates.s3.amazonaws.com", - "https://queue.taskcluster.net/", - "http://ftp.mozilla.org/", - "http://download.mozilla.org/", - "https://archive.mozilla.org/", -] - -DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \ - "{locale}-{from_buildid}-{to_buildid}.partial.mar" - - -def verify_signature(mar, signature): - log.info("Checking %s signature", mar) - m = MarFile(mar, signature_versions=[(1, signature)]) - m.verify_signatures() - - -@redo.retriable() -def download(url, dest, mode=None): - log.debug("Downloading %s to %s", url, dest) - r = requests.get(url) - r.raise_for_status() - - bytes_downloaded = 0 - with open(dest, 'wb') as fd: - for chunk in r.iter_content(4096): - fd.write(chunk) - bytes_downloaded += len(chunk) - - log.debug('Downloaded %s bytes', bytes_downloaded) - if 'content-length' in r.headers: - log.debug('Content-Length: %s bytes', r.headers['content-length']) - if bytes_downloaded != int(r.headers['content-length']): - raise IOError('Unexpected number of bytes downloaded') - - if mode: - log.debug("chmod %o %s", mode, dest) - os.chmod(dest, mode) - - -def unpack(work_env, mar, dest_dir): - os.mkdir(dest_dir) - unwrap_cmd = sh.Command(os.path.join(work_env.workdir, - "unwrap_full_update.pl")) - log.debug("Unwrapping %s", mar) - out = unwrap_cmd(mar, _cwd=dest_dir, _env=work_env.env, _timeout=240, - _err_to_out=True) - if out: - log.debug(out) - - -def find_file(directory, filename): - log.debug("Searching for %s in %s", filename, directory) - for root, dirs, files in os.walk(directory): - if filename in files: - f = os.path.join(root, filename) - log.debug("Found %s", f) - return f - - -def get_option(directory, filename, section, option): - log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory, - filename) - f = find_file(directory, filename) - config = ConfigParser.ConfigParser() - config.read(f) - rv = config.get(section, option) - log.debug("Found %s", rv) - return rv - - -def generate_partial(work_env, from_dir, to_dir, dest_mar, channel_ids, - version): - log.debug("Generating partial %s", dest_mar) - env = work_env.env - env["MOZ_PRODUCT_VERSION"] = version - env["MOZ_CHANNEL_ID"] = channel_ids - make_incremental_update = os.path.join(work_env.workdir, - "make_incremental_update.sh") - out = sh.bash(make_incremental_update, dest_mar, from_dir, to_dir, - _cwd=work_env.workdir, _env=env, _timeout=900, - _err_to_out=True) - if out: - log.debug(out) - - -def get_hash(path, hash_type="sha512"): - h = hashlib.new(hash_type) - with open(path, "rb") as f: - for chunk in iter(functools.partial(f.read, 4096), ''): - h.update(chunk) - return h.hexdigest() - - -class WorkEnv(object): - - def __init__(self): - self.workdir = tempfile.mkdtemp() - - def setup(self): - self.download_unwrap() - self.download_martools() - - def download_unwrap(self): - # unwrap_full_update.pl is not too sensitive to the revision - url = "https://hg.mozilla.org/mozilla-central/raw-file/default/" \ - "tools/update-packaging/unwrap_full_update.pl" - download(url, dest=os.path.join(self.workdir, "unwrap_full_update.pl"), - mode=0o755) - - def download_buildsystem_bits(self, repo, revision): - prefix = "{repo}/raw-file/{revision}/tools/update-packaging" - prefix = prefix.format(repo=repo, revision=revision) - for f in ("make_incremental_update.sh", "common.sh"): - url = "{prefix}/{f}".format(prefix=prefix, f=f) - download(url, dest=os.path.join(self.workdir, f), mode=0o755) - - def download_martools(self): - # TODO: check if the tools have to be branch specific - prefix = "https://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/" \ - "latest-mozilla-central/mar-tools/linux64" - for f in ("mar", "mbsdiff"): - url = "{prefix}/{f}".format(prefix=prefix, f=f) - download(url, dest=os.path.join(self.workdir, f), mode=0o755) - - def cleanup(self): - shutil.rmtree(self.workdir) - - @property - def env(self): - my_env = os.environ.copy() - my_env['LC_ALL'] = 'C' - my_env['MAR'] = os.path.join(self.workdir, "mar") - my_env['MBSDIFF'] = os.path.join(self.workdir, "mbsdiff") - return my_env - - -def verify_allowed_url(mar): - if not any(mar.startswith(prefix) for prefix in ALLOWED_URL_PREFIXES): - raise ValueError("{mar} is not in allowed URL prefixes: {p}".format( - mar=mar, p=ALLOWED_URL_PREFIXES - )) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--artifacts-dir", required=True) - parser.add_argument("--signing-cert", required=True) - parser.add_argument("--task-definition", required=True, - type=argparse.FileType('r')) - parser.add_argument("--filename-template", - default=DEFAULT_FILENAME_TEMPLATE) - parser.add_argument("--no-freshclam", action="store_true", default=False, - help="Do not refresh ClamAV DB") - parser.add_argument("-q", "--quiet", dest="log_level", - action="store_const", const=logging.WARNING, - default=logging.DEBUG) - args = parser.parse_args() - - logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", - level=args.log_level) - task = json.load(args.task_definition) - # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema - - if args.no_freshclam: - log.info("Skipping freshclam") - else: - log.info("Refreshing clamav db...") - try: - redo.retry(lambda: - sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True)) - log.info("Done.") - except sh.ErrorReturnCode: - log.warning("Freshclam failed, skipping DB update") - manifest = [] - for e in task["extra"]["funsize"]["partials"]: - for mar in (e["from_mar"], e["to_mar"]): - verify_allowed_url(mar) - - work_env = WorkEnv() - # TODO: run setup once - work_env.setup() - complete_mars = {} - for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])): - dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type)) - unpack_dir = os.path.join(work_env.workdir, mar_type) - download(f, dest) - if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"): - verify_signature(dest, args.signing_cert) - complete_mars["%s_size" % mar_type] = os.path.getsize(dest) - complete_mars["%s_hash" % mar_type] = get_hash(dest) - unpack(work_env, dest, unpack_dir) - log.info("AV-scanning %s ...", unpack_dir) - sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True) - log.info("Done.") - - path = os.path.join(work_env.workdir, "to") - from_path = os.path.join(work_env.workdir, "from") - mar_data = { - "ACCEPTED_MAR_CHANNEL_IDS": get_option( - path, filename="update-settings.ini", section="Settings", - option="ACCEPTED_MAR_CHANNEL_IDS"), - "version": get_option(path, filename="application.ini", - section="App", option="Version"), - "to_buildid": get_option(path, filename="application.ini", - section="App", option="BuildID"), - "from_buildid": get_option(from_path, filename="application.ini", - section="App", option="BuildID"), - "appName": get_option(from_path, filename="application.ini", - section="App", option="Name"), - # Use Gecko repo and rev from platform.ini, not application.ini - "repo": get_option(path, filename="platform.ini", section="Build", - option="SourceRepository"), - "revision": get_option(path, filename="platform.ini", - section="Build", option="SourceStamp"), - "from_mar": e["from_mar"], - "to_mar": e["to_mar"], - "platform": e["platform"], - "locale": e["locale"], - } - # Override ACCEPTED_MAR_CHANNEL_IDS if needed - if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ: - mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ["ACCEPTED_MAR_CHANNEL_IDS"] - for field in ("update_number", "previousVersion", - "previousBuildNumber", "toVersion", - "toBuildNumber"): - if field in e: - mar_data[field] = e[field] - mar_data.update(complete_mars) - # if branch not set explicitly use repo-name - mar_data["branch"] = e.get("branch", - mar_data["repo"].rstrip("/").split("/")[-1]) - mar_name = args.filename_template.format(**mar_data) - mar_data["mar"] = mar_name - dest_mar = os.path.join(work_env.workdir, mar_name) - # TODO: download these once - work_env.download_buildsystem_bits(repo=mar_data["repo"], - revision=mar_data["revision"]) - generate_partial(work_env, from_path, path, dest_mar, - mar_data["ACCEPTED_MAR_CHANNEL_IDS"], - mar_data["version"]) - mar_data["size"] = os.path.getsize(dest_mar) - mar_data["hash"] = get_hash(dest_mar) - - shutil.copy(dest_mar, args.artifacts_dir) - work_env.cleanup() - manifest.append(mar_data) - manifest_file = os.path.join(args.artifacts_dir, "manifest.json") - with open(manifest_file, "w") as fp: - json.dump(manifest, fp, indent=2, sort_keys=True) - - -if __name__ == '__main__': - main() diff -Nru firefox-52.0+build2/release/docker/funsize-update-generator/scripts/mbsdiff_hook.sh firefox-52.0.1+build2/release/docker/funsize-update-generator/scripts/mbsdiff_hook.sh --- firefox-52.0+build2/release/docker/funsize-update-generator/scripts/mbsdiff_hook.sh 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/release/docker/funsize-update-generator/scripts/mbsdiff_hook.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,135 +0,0 @@ -#!/bin/bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# -# This tool contains functions that are to be used to handle/enable funsize -# Author: Mihai Tabara -# - -HOOK= -SERVER_URL= -LOCAL_CACHE_DIR= - -getsha512(){ - echo "$(openssl sha512 "${1}" | awk '{print $2}')" -} - -print_usage(){ - echo "$(basename $0) -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] PATH-FROM-URL PATH-TO-URL PATH-PATCH" - echo "Script that saves/retrieves from cache presumptive patches as args" - echo "" - echo "-A SERVER-URL - host where to send the files" - echo "-c LOCAL-CACHE-DIR-PATH local path to which patches are cached" - echo "-g pre hook - tests whether patch already in cache" - echo "-u post hook - upload patch to cache for future use" - echo "" - echo "PATH-FROM-URL : path on disk for source file" - echo "PATH-TO-URL : path on disk for destination file" - echo "PATH-PATCH : path on disk for patch between source and destination" -} - -upload_patch(){ - sha_from=`getsha512 "$1"` - sha_to=`getsha512 "$2"` - patch_path="$3" - - # save to local cache first - if [ -n "$LOCAL_CACHE_DIR" ]; then - local_cmd="mkdir -p "$LOCAL_CACHE_DIR/$sha_from"" - if `$local_cmd` >&2; then - cp -avf "$patch_path" "$LOCAL_CACHE_DIR/$sha_from/$sha_to" - echo "$patch_path saved on local cache!" - fi - fi - # The remote cache implementation is not used. The code is for usage - # reference only. - return 0 - - # send it over to funsize - cmd="curl -sSw %{http_code} -o /dev/null -X POST $SERVER_URL -F sha_from="$sha_from" -F sha_to="$sha_to" -F patch_file="@$patch_path"" - ret_code=`$cmd` - - if [ $ret_code -eq 200 ]; then - echo "$patch_path Successful uploaded to funsize!" - return 0 - fi - - echo "$patch_path Failed to be uploaded to funsize!" - return 1 -} - -get_patch(){ - sha_from=`getsha512 "$1"` - sha_to=`getsha512 "$2"` - destination_file="$3" - tmp_file="$destination_file.tmp" - - # try to retrieve from local cache first - if [ -r "$LOCAL_CACHE_DIR/$sha_from/$sha_to" ]; then - cp -avf "$LOCAL_CACHE_DIR/$sha_from/$sha_to" "$destination_file" - echo "Successful retrieved $destination_file from local cache!" - return 0 - else - echo "File is not in the locale cache" - return 1 - fi - # The remote cache implementation is not used. The code is for usage - # reference only. - - # if unsuccessful, try to retrieve from funsize - cmd="curl -LsSGw %{http_code} $SERVER_URL/$sha_from/$sha_to -o $tmp_file" - ret_code=`$cmd` - - if [ $ret_code -eq 200 ]; then - mv "$tmp_file" "$destination_file" - echo "Successful retrieved $destination_file from funsize!" - return 0 - fi - - rm -f "$tmp_file" - echo "Failed to retrieve $destination_file from funsize!" - return 1 -} - -OPTIND=1 - -while getopts ":A:c:gu" option; do - case $option in - A) - SERVER_URL="$OPTARG" - ;; - c) - LOCAL_CACHE_DIR="$OPTARG" - ;; - g) - HOOK="PRE" - ;; - u) - HOOK="POST" - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - print_usage - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - print_usage - exit 1 - ;; - *) - echo "Unimplemented option: -$OPTARG" >&2 - print_usage - exit 1 - ;; - esac -done -shift $((OPTIND-1)) - -if [ "$HOOK" == "PRE" ]; then - get_patch "$1" "$2" "$3" -elif [ "$HOOK" == "POST" ]; then - upload_patch "$1" "$2" "$3" -fi diff -Nru firefox-52.0+build2/SOURCE_CHANGESET firefox-52.0.1+build2/SOURCE_CHANGESET --- firefox-52.0+build2/SOURCE_CHANGESET 2017-03-02 23:36:43.000000000 +0000 +++ firefox-52.0.1+build2/SOURCE_CHANGESET 2017-03-17 13:41:16.000000000 +0000 @@ -1 +1 @@ -44d6a57ab554308585a67a13035d31b264be781e \ No newline at end of file +2f2b4a119565e9b5691187ee5fbe91881c90b249 \ No newline at end of file diff -Nru firefox-52.0+build2/testing/docker/beet-mover/Dockerfile firefox-52.0.1+build2/testing/docker/beet-mover/Dockerfile --- firefox-52.0+build2/testing/docker/beet-mover/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/beet-mover/Dockerfile 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,22 @@ +FROM ubuntu:vivid + +RUN apt-get -q update \ + && apt-get install --yes -q \ + mercurial \ + python-dev \ + python-pip \ + python-virtualenv \ + libffi-dev \ + libssl-dev \ + libyaml-dev \ + libmysqlclient-dev \ + clamav \ + clamav-freshclam \ + curl \ + wget \ + && apt-get clean + +COPY requirements.txt /tmp/ +RUN pip install -r /tmp/requirements.txt +# Freshclam may be flaky, retry if it fails +RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done diff -Nru firefox-52.0+build2/testing/docker/beet-mover/requirements.txt firefox-52.0.1+build2/testing/docker/beet-mover/requirements.txt --- firefox-52.0+build2/testing/docker/beet-mover/requirements.txt 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/beet-mover/requirements.txt 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,2 @@ +sh +redo diff -Nru firefox-52.0+build2/testing/docker/firefox-snap/distribution.ini firefox-52.0.1+build2/testing/docker/firefox-snap/distribution.ini --- firefox-52.0+build2/testing/docker/firefox-snap/distribution.ini 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/firefox-snap/distribution.ini 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,9 @@ +[Global] +id=mozilla-snap +version=1.0 +about=Mozilla Firefox Snap + +[Preferences] +app.update.enabled=false +intl.locale.matchOS=true +browser.shell.checkDefaultBrowser=false diff -Nru firefox-52.0+build2/testing/docker/firefox-snap/Dockerfile firefox-52.0.1+build2/testing/docker/firefox-snap/Dockerfile --- firefox-52.0+build2/testing/docker/firefox-snap/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/firefox-snap/Dockerfile 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,3 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -qy snapcraft bzip2 curl && apt-get clean diff -Nru firefox-52.0+build2/testing/docker/firefox-snap/Makefile firefox-52.0.1+build2/testing/docker/firefox-snap/Makefile --- firefox-52.0+build2/testing/docker/firefox-snap/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/firefox-snap/Makefile 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,12 @@ +DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) +IMAGE_NAME = firefox-snapcraft +FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) + +build: + docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . + +push: + docker push $(FULL_IMAGE_NAME):latest + +pull: + docker pull $(FULL_IMAGE_NAME):latest diff -Nru firefox-52.0+build2/testing/docker/firefox-snap/runme.sh firefox-52.0.1+build2/testing/docker/firefox-snap/runme.sh --- firefox-52.0+build2/testing/docker/firefox-snap/runme.sh 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/firefox-snap/runme.sh 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,66 @@ +#!/bin/sh + +set -xe + +# Required env variables +test $VERSION +test $BUILD_NUMBER +test $CANDIDATES_DIR + +# Optional env variables +: WORKSPACE ${WORKSPACE:=/home/worker/workspace} +: ARTIFACTS_DIR ${ARTIFACTS_DIR:=/home/worker/artifacts} + + +TARGET="firefox-${VERSION}.snap" + +mkdir -p "$ARTIFACTS_DIR" +rm -rf "${WORKSPACE}/source" && mkdir -p "${WORKSPACE}/source/opt" "${WORKSPACE}/source/usr/bin" + +CURL="curl --location --retry 10 --retry-delay 10" + +# Download and extract en-US linux64 binary +$CURL -o "${WORKSPACE}/firefox.tar.bz2" \ + "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/firefox-${VERSION}.tar.bz2" + +tar -C "${WORKSPACE}/source/opt" -xf "${WORKSPACE}/firefox.tar.bz2" +mkdir -p "${WORKSPACE}/source/opt/firefox/distribution/extensions" +cp -v distribution.ini "${WORKSPACE}/source/opt/firefox/distribution/" + +# Use release-specific list of locales to fetch L10N XPIs +$CURL -o "${WORKSPACE}/l10n_changesets.txt" "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/l10n_changesets.txt" +cat "${WORKSPACE}/l10n_changesets.txt" + +for locale in $(grep -v ja-JP-mac "${WORKSPACE}/l10n_changesets.txt" | awk '{print $1}'); do + $CURL -o "${WORKSPACE}/source/opt/firefox/distribution/extensions/langpack-${locale}@firefox.mozilla.org.xpi" \ + "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" +done + +# Symlink firefox binary to /usr/bin to make it available in PATH +ln -s ../../opt/firefox/firefox "${WORKSPACE}/source/usr/bin" + +# Generate snapcraft manifest +sed -e "s/@VERSION@/${VERSION}/g" -e "s/@BUILD_NUMBER@/${BUILD_NUMBER}/g" snapcraft.yaml.in > ${WORKSPACE}/snapcraft.yaml +cd ${WORKSPACE} +snapcraft + +mv *.snap "$ARTIFACTS_DIR/$TARGET" + +cd $ARTIFACTS_DIR + +# Generate checksums file +size=$(stat --printf="%s" $ARTIFACTS_DIR/$TARGET) +sha=$(sha512sum $ARTIFACTS_DIR/$TARGET | awk '{print $1}') +echo "$sha sha512 $size $TARGET" > $TARGET.checksums + +echo "Generating signing manifest" +hash=$(sha512sum $TARGET.checksums | awk '{print $1}') + +cat << EOF > signing_manifest.json +[{"file_to_sign": "$TARGET.checksums", "hash": "$hash"}] +EOF + +# For posterity +find . -ls +cat $TARGET.checksums +cat signing_manifest.json diff -Nru firefox-52.0+build2/testing/docker/firefox-snap/snapcraft.yaml.in firefox-52.0.1+build2/testing/docker/firefox-snap/snapcraft.yaml.in --- firefox-52.0+build2/testing/docker/firefox-snap/snapcraft.yaml.in 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/firefox-snap/snapcraft.yaml.in 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,37 @@ +name: firefox +version: @VERSION@-@BUILD_NUMBER@ +summary: Mozilla Firefox web browser +description: Firefox is a powerful, extensible web browser with support for modern web application technologies. +confinement: strict + +apps: + firefox: + command: desktop-launch firefox + plugs: + - unity7 + - network + - home + - x11 + - opengl + - pulseaudio + - gsettings + - camera + - browser-sandbox + +plugs: + browser-sandbox: + interface: browser-support + allow-sandbox: true + +parts: + firefox: + plugin: dump + source: source + stage-packages: + - libxt6 + - libdbus-glib-1-2 + - libasound2 + - libpulse0 + - libgl1-mesa-dri + - libgl1-mesa-glx + after: [desktop-gtk3] diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/dep.pubkey firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/dep.pubkey --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/dep.pubkey 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/dep.pubkey 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzjHSobdeiQ3JHP/cCIOp +WaX9y12rL5mIo9OR9bpqEZdD0yXJJJeZA887Mv8slqsM+qObMUpKvfEE6zyYPIZJ +ANib31neI5BBYHhfhf2f5EnkilSYlmU3Gx+uRsmsdt58PpYe124tOAGgca/8bUy3 +eb6kUUTwvMI0oWQuPkGUaoHVQyj/bBMTrIkyF3UbfFtiX/SfOPvIoabNUe+pQHUe +pqC2+RxzDGj+shTq/hYhtXlptFzsEEb2+0foLy0MY8C30dP2QqbM2iavvr/P8OcS +Gm3H0TQcRzIEBzvPcIjiZi1nQj/r/3TlYRNCjuYT/HsNLXrB/U5Tc990jjAUJxdH +0wIDAQAB +-----END PUBLIC KEY----- diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/Dockerfile firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/Dockerfile --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/Dockerfile 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,33 @@ +FROM ubuntu:vivid +MAINTAINER Rail Aliiev + +# Required software +ENV DEBIAN_FRONTEND noninteractive +# Chain apt-get commands with apt-get clean in a single docker RUN +# to make sure that files are removed within a single docker layer +RUN apt-get update -q && \ + apt-get install -yyq --no-install-recommends \ + python mercurial curl python-boto python-setuptools python-cryptography && \ + apt-get clean + +COPY requirements.txt /tmp/ +# python-pip installs a lot of dependencies increasing the size of an image +# drastically. +RUN easy_install pip +RUN pip install -r /tmp/requirements.txt + +RUN hg clone https://hg.mozilla.org/build/tools /home/worker/tools + +RUN useradd -d /home/worker -s /bin/bash -m worker + +RUN mkdir /home/worker/bin +COPY scripts/* /home/worker/bin/ +RUN mkdir /home/worker/keys +COPY *.pubkey /home/worker/keys/ +COPY runme.sh /runme.sh +RUN chmod 755 /home/worker/bin/* /runme.sh + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/Makefile firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/Makefile --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/Makefile 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,17 @@ +DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) +IMAGE_NAME = funsize-balrog-submitter +FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) + +build: + docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . + +push: + docker push $(FULL_IMAGE_NAME):latest + +pull: + docker pull $(FULL_IMAGE_NAME):latest + +update_pubkeys: + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/nightly.pubkey firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/nightly.pubkey --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/nightly.pubkey 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/nightly.pubkey 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4j/IS3gWbyVVnWn4ZRnC +Fuzb6VAaHa0I+4E504ekhVAhbKlSfBstkLbXajdjUVAJpn02zWnOaTl5KAdpDpIp +SkdA4mK20ej3/Ij7gIt8IwaX+ArXL8mP84pxDn5BgaNADm3206Z6YQzc/TDYu529 +qkDFmLqNUVRJAhPO+qqhKHIcVGh8HUHXN6XV1qOFip+UU0M474jAGgurVmAv8Rh7 +VvM0v5KmB6V6WHwM5gwjg2yRY/o+xYIsNeSes9rpp+MOs/RnUA6LI4WZGY4YahvX +VclIXBDgbWPYtojexIJkmYj8JIIRsh3eCsrRRe14fq7cBurp3CxBYMlDHf0RUoaq +hQIDAQAB +-----END PUBLIC KEY----- diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/release.pubkey firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/release.pubkey --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/release.pubkey 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/release.pubkey 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvH4r94FpQ0gvr1hhTfV9 +NUeWPJ5CN6TZRq7v/Dc4nkJ1J4IP1B3UEii34tcNKpy1nKupiZuTT6T1zQYT+z5x +3UkDF9qQboQ8RNb/BEz/cN3on/LTEnZ7YSraRL11M6cEB8mvmJxddCEquwqccRbs +Usp8WUB7uRv1w6Anley7N9F/LE1iLPwJasZypRnzWb3aYsJy0cMFOYy+OXVdpktn +qYqlNIjnt84u4Nil6UXnBbIJNUVOCY8wOFClNvVpubjPkWK1gtdWy3x/hJU5RpAO +K9cnHxq4M/I4SUWTWO3r7yweQiHG4Jyoc7sP1jkwjBkSG93sDEycfwOdOoZft3wN +sQIDAQAB +-----END PUBLIC KEY----- diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/requirements.txt firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/requirements.txt --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/requirements.txt 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/requirements.txt 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1 @@ +mar==1.2 diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/runme.sh firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/runme.sh --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/runme.sh 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/runme.sh 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,22 @@ +#!/bin/bash + +set -xe + +test $PARENT_TASK_ARTIFACTS_URL_PREFIX +test $BALROG_API_ROOT +test $SIGNING_CERT + +ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +curl --location --retry 10 --retry-delay 10 -o "$ARTIFACTS_DIR/manifest.json" \ + "$PARENT_TASK_ARTIFACTS_URL_PREFIX/manifest.json" + +cat "$ARTIFACTS_DIR/manifest.json" +python /home/worker/bin/funsize-balrog-submitter.py \ + --artifacts-url-prefix "$PARENT_TASK_ARTIFACTS_URL_PREFIX" \ + --manifest "$ARTIFACTS_DIR/manifest.json" \ + -a "$BALROG_API_ROOT" \ + --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + --verbose \ + $EXTRA_BALROG_SUBMITTER_PARAMS diff -Nru firefox-52.0+build2/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py --- firefox-52.0+build2/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,206 @@ +#!/usr/bin/env python +import site +import os +import logging +import argparse +import json +import hashlib +import requests +import tempfile +from boto.s3.connection import S3Connection +from mardor.marfile import MarFile + +site.addsitedir("/home/worker/tools/lib/python") + +from balrog.submitter.cli import NightlySubmitterV4, ReleaseSubmitterV4 +from util.retry import retry, retriable + +log = logging.getLogger(__name__) + + +def get_hash(content, hash_type="md5"): + h = hashlib.new(hash_type) + h.update(content) + return h.hexdigest() + + +@retriable() +def download(url, dest, mode=None): + log.debug("Downloading %s to %s", url, dest) + r = requests.get(url) + r.raise_for_status() + + bytes_downloaded = 0 + with open(dest, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + bytes_downloaded += len(chunk) + + log.debug('Downloaded %s bytes', bytes_downloaded) + if 'content-length' in r.headers: + log.debug('Content-Length: %s bytes', r.headers['content-length']) + if bytes_downloaded != int(r.headers['content-length']): + raise IOError('Unexpected number of bytes downloaded') + + if mode: + log.debug("chmod %o %s", mode, dest) + os.chmod(dest, mode) + + +def verify_signature(mar, signature): + log.info("Checking %s signature", mar) + m = MarFile(mar, signature_versions=[(1, signature)]) + m.verify_signatures() + + +def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key, + mar_url, mar_dest, signing_cert): + conn = S3Connection(aws_access_key_id, aws_secret_access_key) + bucket = conn.get_bucket(bucket_name) + _, dest = tempfile.mkstemp() + log.info("Downloading %s to %s...", mar_url, dest) + download(mar_url, dest) + log.info("Verifying the signature...") + if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"): + verify_signature(dest, signing_cert) + for name in possible_names(mar_dest, 10): + log.info("Checking if %s already exists", name) + key = bucket.get_key(name) + if not key: + log.info("Uploading to %s...", name) + key = bucket.new_key(name) + # There is a chance for race condition here. To avoid it we check + # the return value with replace=False. It should be not None. + length = key.set_contents_from_filename(dest, replace=False) + if length is None: + log.warn("Name race condition using %s, trying again...", name) + continue + else: + # key.make_public() may lead to race conditions, because + # it doesn't pass version_id, so it may not set permissions + bucket.set_canned_acl(acl_str='public-read', key_name=name, + version_id=key.version_id) + # Use explicit version_id to avoid using "latest" version + return key.generate_url(expires_in=0, query_auth=False, + version_id=key.version_id) + else: + if get_hash(key.get_contents_as_string()) == \ + get_hash(open(dest).read()): + log.info("%s has the same MD5 checksum, not uploading...", + name) + return key.generate_url(expires_in=0, query_auth=False, + version_id=key.version_id) + log.info("%s already exists with different checksum, " + "trying another one...", name) + + raise RuntimeError("Cannot generate a unique name for %s", mar_dest) + + +def possible_names(initial_name, amount): + """Generate names appending counter before extension""" + prefix, ext = os.path.splitext(initial_name) + return [initial_name] + ["{}-{}{}".format(prefix, n, ext) for n in + range(1, amount + 1)] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts-url-prefix", required=True, + help="URL prefix for MAR") + parser.add_argument("--manifest", required=True) + parser.add_argument("-a", "--api-root", required=True, + help="Balrog API root") + parser.add_argument("-d", "--dummy", action="store_true", + help="Add '-dummy' suffix to branch name") + parser.add_argument("--signing-cert", required=True) + parser.add_argument("-v", "--verbose", action="store_const", + dest="loglevel", const=logging.DEBUG, + default=logging.INFO) + args = parser.parse_args() + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", + level=args.loglevel) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("boto").setLevel(logging.WARNING) + + balrog_username = os.environ.get("BALROG_USERNAME") + balrog_password = os.environ.get("BALROG_PASSWORD") + if not balrog_username and not balrog_password: + raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment " + "variables should be set") + + s3_bucket = os.environ.get("S3_BUCKET") + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID") + aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY") + if not (s3_bucket and aws_access_key_id and aws_secret_access_key): + log.warn("Skipping S3 uploads...") + uploads_enabled = False + else: + uploads_enabled = True + + manifest = json.load(open(args.manifest)) + auth = (balrog_username, balrog_password) + + for e in manifest: + complete_info = [{ + "hash": e["to_hash"], + "size": e["to_size"], + }] + partial_info = [{ + "hash": e["hash"], + "size": e["size"], + }] + + if "previousVersion" in e and "previousBuildNumber" in e: + log.info("Release style balrog submission") + partial_info[0]["previousVersion"] = e["previousVersion"] + partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"] + submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth, + dummy=args.dummy) + retry(lambda: submitter.run( + platform=e["platform"], productName=e["appName"], + version=e["toVersion"], + build_number=e["toBuildNumber"], + appVersion=e["version"], extVersion=e["version"], + buildID=e["to_buildid"], locale=e["locale"], + hashFunction='sha512', + partialInfo=partial_info, completeInfo=complete_info, + )) + elif "from_buildid" in e and uploads_enabled: + log.info("Nightly style balrog submission") + partial_mar_url = "{}/{}".format(args.artifacts_url_prefix, + e["mar"]) + complete_mar_url = e["to_mar"] + dest_prefix = "{branch}/{buildid}".format( + branch=e["branch"], buildid=e["to_buildid"]) + partial_mar_dest = "{}/{}".format(dest_prefix, e["mar"]) + complete_mar_filename = "{appName}-{branch}-{version}-" \ + "{platform}-{locale}.complete.mar" + complete_mar_filename = complete_mar_filename.format( + appName=e["appName"], branch=e["branch"], + version=e["version"], platform=e["platform"], + locale=e["locale"] + ) + complete_mar_dest = "{}/{}".format(dest_prefix, + complete_mar_filename) + partial_info[0]["url"] = verify_copy_to_s3( + s3_bucket, aws_access_key_id, aws_secret_access_key, + partial_mar_url, partial_mar_dest, args.signing_cert) + complete_info[0]["url"] = verify_copy_to_s3( + s3_bucket, aws_access_key_id, aws_secret_access_key, + complete_mar_url, complete_mar_dest, args.signing_cert) + partial_info[0]["from_buildid"] = e["from_buildid"] + submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth, + dummy=args.dummy) + retry(lambda: submitter.run( + platform=e["platform"], buildID=e["to_buildid"], + productName=e["appName"], branch=e["branch"], + appVersion=e["version"], locale=e["locale"], + hashFunction='sha512', extVersion=e["version"], + partialInfo=partial_info, completeInfo=complete_info), + attempts=30, sleeptime=10, max_sleeptime=60, + ) + else: + raise RuntimeError("Cannot determine Balrog submission style") + +if __name__ == '__main__': + main() diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/dep.pubkey firefox-52.0.1+build2/testing/docker/funsize-update-generator/dep.pubkey --- firefox-52.0+build2/testing/docker/funsize-update-generator/dep.pubkey 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/dep.pubkey 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzjHSobdeiQ3JHP/cCIOp +WaX9y12rL5mIo9OR9bpqEZdD0yXJJJeZA887Mv8slqsM+qObMUpKvfEE6zyYPIZJ +ANib31neI5BBYHhfhf2f5EnkilSYlmU3Gx+uRsmsdt58PpYe124tOAGgca/8bUy3 +eb6kUUTwvMI0oWQuPkGUaoHVQyj/bBMTrIkyF3UbfFtiX/SfOPvIoabNUe+pQHUe +pqC2+RxzDGj+shTq/hYhtXlptFzsEEb2+0foLy0MY8C30dP2QqbM2iavvr/P8OcS +Gm3H0TQcRzIEBzvPcIjiZi1nQj/r/3TlYRNCjuYT/HsNLXrB/U5Tc990jjAUJxdH +0wIDAQAB +-----END PUBLIC KEY----- diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/Dockerfile firefox-52.0.1+build2/testing/docker/funsize-update-generator/Dockerfile --- firefox-52.0+build2/testing/docker/funsize-update-generator/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/Dockerfile 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,33 @@ +FROM ubuntu:vivid +MAINTAINER Rail Aliiev + +# Required software +ENV DEBIAN_FRONTEND noninteractive +# Chain apt-get commands with apt-get clean in a single docker RUN +# to make sure that files are removed within a single docker layer +RUN apt-get update -q && \ + apt-get install -yyq --no-install-recommends \ + python python-setuptools python-cryptography libgetopt-simple-perl \ + bzip2 clamav clamav-freshclam python-requests python-sh curl && \ + apt-get clean +RUN useradd -d /home/worker -s /bin/bash -m worker +COPY requirements.txt /tmp/ +# python-pip installs a lot of dependencies increasing the size of an image +# drastically. Using easy_install saves us almost 200M. +RUN easy_install pip +RUN pip install -r /tmp/requirements.txt + +# scripts +RUN mkdir /home/worker/bin +COPY scripts/* /home/worker/bin/ +COPY runme.sh /runme.sh +RUN chmod 755 /home/worker/bin/* /runme.sh +RUN mkdir /home/worker/keys +COPY *.pubkey /home/worker/keys/ +# Freshclam may be flaky, retry if it fails +RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/Makefile firefox-52.0.1+build2/testing/docker/funsize-update-generator/Makefile --- firefox-52.0+build2/testing/docker/funsize-update-generator/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/Makefile 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,17 @@ +DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) +IMAGE_NAME = funsize-update-generator +FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) + +build: + docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . + +push: + docker push $(FULL_IMAGE_NAME):latest + +pull: + docker pull $(FULL_IMAGE_NAME):latest + +update_pubkeys: + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/nightly.pubkey firefox-52.0.1+build2/testing/docker/funsize-update-generator/nightly.pubkey --- firefox-52.0+build2/testing/docker/funsize-update-generator/nightly.pubkey 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/nightly.pubkey 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4j/IS3gWbyVVnWn4ZRnC +Fuzb6VAaHa0I+4E504ekhVAhbKlSfBstkLbXajdjUVAJpn02zWnOaTl5KAdpDpIp +SkdA4mK20ej3/Ij7gIt8IwaX+ArXL8mP84pxDn5BgaNADm3206Z6YQzc/TDYu529 +qkDFmLqNUVRJAhPO+qqhKHIcVGh8HUHXN6XV1qOFip+UU0M474jAGgurVmAv8Rh7 +VvM0v5KmB6V6WHwM5gwjg2yRY/o+xYIsNeSes9rpp+MOs/RnUA6LI4WZGY4YahvX +VclIXBDgbWPYtojexIJkmYj8JIIRsh3eCsrRRe14fq7cBurp3CxBYMlDHf0RUoaq +hQIDAQAB +-----END PUBLIC KEY----- diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/release.pubkey firefox-52.0.1+build2/testing/docker/funsize-update-generator/release.pubkey --- firefox-52.0+build2/testing/docker/funsize-update-generator/release.pubkey 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/release.pubkey 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvH4r94FpQ0gvr1hhTfV9 +NUeWPJ5CN6TZRq7v/Dc4nkJ1J4IP1B3UEii34tcNKpy1nKupiZuTT6T1zQYT+z5x +3UkDF9qQboQ8RNb/BEz/cN3on/LTEnZ7YSraRL11M6cEB8mvmJxddCEquwqccRbs +Usp8WUB7uRv1w6Anley7N9F/LE1iLPwJasZypRnzWb3aYsJy0cMFOYy+OXVdpktn +qYqlNIjnt84u4Nil6UXnBbIJNUVOCY8wOFClNvVpubjPkWK1gtdWy3x/hJU5RpAO +K9cnHxq4M/I4SUWTWO3r7yweQiHG4Jyoc7sP1jkwjBkSG93sDEycfwOdOoZft3wN +sQIDAQAB +-----END PUBLIC KEY----- diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/requirements.txt firefox-52.0.1+build2/testing/docker/funsize-update-generator/requirements.txt --- firefox-52.0+build2/testing/docker/funsize-update-generator/requirements.txt 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/requirements.txt 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,2 @@ +mar==1.2 +redo diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/runme.sh firefox-52.0.1+build2/testing/docker/funsize-update-generator/runme.sh --- firefox-52.0+build2/testing/docker/funsize-update-generator/runme.sh 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/runme.sh 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,25 @@ +#!/bin/sh + +set -xe + +test $TASK_ID +test $SIGNING_CERT + +ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \ + "https://queue.taskcluster.net/v1/task/$TASK_ID" + +# enable locale cache +export MBSDIFF_HOOK="/home/worker/bin/mbsdiff_hook.sh -c /tmp/fs-cache" + +if [ ! -z $FILENAME_TEMPLATE ]; then + EXTRA_PARAMS="--filename-template $FILENAME_TEMPLATE $EXTRA_PARAMS" +fi + +/home/worker/bin/funsize.py \ + --artifacts-dir "$ARTIFACTS_DIR" \ + --task-definition /home/worker/task.json \ + --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + $EXTRA_PARAMS diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/scripts/funsize.py firefox-52.0.1+build2/testing/docker/funsize-update-generator/scripts/funsize.py --- firefox-52.0+build2/testing/docker/funsize-update-generator/scripts/funsize.py 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/scripts/funsize.py 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,275 @@ +#!/usr/bin/env python + +import ConfigParser +import argparse +import functools +import hashlib +import json +import logging +import os +import shutil +import tempfile +import requests +import sh + +import redo +from mardor.marfile import MarFile + +log = logging.getLogger(__name__) +ALLOWED_URL_PREFIXES = [ + "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/", + "http://download.cdn.mozilla.net/pub/firefox/nightly/", + "https://mozilla-nightly-updates.s3.amazonaws.com", + "https://queue.taskcluster.net/", + "http://ftp.mozilla.org/", + "http://download.mozilla.org/", + "https://archive.mozilla.org/", +] + +DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \ + "{locale}-{from_buildid}-{to_buildid}.partial.mar" + + +def verify_signature(mar, signature): + log.info("Checking %s signature", mar) + m = MarFile(mar, signature_versions=[(1, signature)]) + m.verify_signatures() + + +@redo.retriable() +def download(url, dest, mode=None): + log.debug("Downloading %s to %s", url, dest) + r = requests.get(url) + r.raise_for_status() + + bytes_downloaded = 0 + with open(dest, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + bytes_downloaded += len(chunk) + + log.debug('Downloaded %s bytes', bytes_downloaded) + if 'content-length' in r.headers: + log.debug('Content-Length: %s bytes', r.headers['content-length']) + if bytes_downloaded != int(r.headers['content-length']): + raise IOError('Unexpected number of bytes downloaded') + + if mode: + log.debug("chmod %o %s", mode, dest) + os.chmod(dest, mode) + + +def unpack(work_env, mar, dest_dir): + os.mkdir(dest_dir) + unwrap_cmd = sh.Command(os.path.join(work_env.workdir, + "unwrap_full_update.pl")) + log.debug("Unwrapping %s", mar) + out = unwrap_cmd(mar, _cwd=dest_dir, _env=work_env.env, _timeout=240, + _err_to_out=True) + if out: + log.debug(out) + + +def find_file(directory, filename): + log.debug("Searching for %s in %s", filename, directory) + for root, dirs, files in os.walk(directory): + if filename in files: + f = os.path.join(root, filename) + log.debug("Found %s", f) + return f + + +def get_option(directory, filename, section, option): + log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory, + filename) + f = find_file(directory, filename) + config = ConfigParser.ConfigParser() + config.read(f) + rv = config.get(section, option) + log.debug("Found %s", rv) + return rv + + +def generate_partial(work_env, from_dir, to_dir, dest_mar, channel_ids, + version): + log.debug("Generating partial %s", dest_mar) + env = work_env.env + env["MOZ_PRODUCT_VERSION"] = version + env["MOZ_CHANNEL_ID"] = channel_ids + make_incremental_update = os.path.join(work_env.workdir, + "make_incremental_update.sh") + out = sh.bash(make_incremental_update, dest_mar, from_dir, to_dir, + _cwd=work_env.workdir, _env=env, _timeout=900, + _err_to_out=True) + if out: + log.debug(out) + + +def get_hash(path, hash_type="sha512"): + h = hashlib.new(hash_type) + with open(path, "rb") as f: + for chunk in iter(functools.partial(f.read, 4096), ''): + h.update(chunk) + return h.hexdigest() + + +class WorkEnv(object): + + def __init__(self): + self.workdir = tempfile.mkdtemp() + + def setup(self): + self.download_unwrap() + self.download_martools() + + def download_unwrap(self): + # unwrap_full_update.pl is not too sensitive to the revision + url = "https://hg.mozilla.org/mozilla-central/raw-file/default/" \ + "tools/update-packaging/unwrap_full_update.pl" + download(url, dest=os.path.join(self.workdir, "unwrap_full_update.pl"), + mode=0o755) + + def download_buildsystem_bits(self, repo, revision): + prefix = "{repo}/raw-file/{revision}/tools/update-packaging" + prefix = prefix.format(repo=repo, revision=revision) + for f in ("make_incremental_update.sh", "common.sh"): + url = "{prefix}/{f}".format(prefix=prefix, f=f) + download(url, dest=os.path.join(self.workdir, f), mode=0o755) + + def download_martools(self): + # TODO: check if the tools have to be branch specific + prefix = "https://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/" \ + "latest-mozilla-central/mar-tools/linux64" + for f in ("mar", "mbsdiff"): + url = "{prefix}/{f}".format(prefix=prefix, f=f) + download(url, dest=os.path.join(self.workdir, f), mode=0o755) + + def cleanup(self): + shutil.rmtree(self.workdir) + + @property + def env(self): + my_env = os.environ.copy() + my_env['LC_ALL'] = 'C' + my_env['MAR'] = os.path.join(self.workdir, "mar") + my_env['MBSDIFF'] = os.path.join(self.workdir, "mbsdiff") + return my_env + + +def verify_allowed_url(mar): + if not any(mar.startswith(prefix) for prefix in ALLOWED_URL_PREFIXES): + raise ValueError("{mar} is not in allowed URL prefixes: {p}".format( + mar=mar, p=ALLOWED_URL_PREFIXES + )) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts-dir", required=True) + parser.add_argument("--signing-cert", required=True) + parser.add_argument("--task-definition", required=True, + type=argparse.FileType('r')) + parser.add_argument("--filename-template", + default=DEFAULT_FILENAME_TEMPLATE) + parser.add_argument("--no-freshclam", action="store_true", default=False, + help="Do not refresh ClamAV DB") + parser.add_argument("-q", "--quiet", dest="log_level", + action="store_const", const=logging.WARNING, + default=logging.DEBUG) + args = parser.parse_args() + + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", + level=args.log_level) + task = json.load(args.task_definition) + # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema + + if args.no_freshclam: + log.info("Skipping freshclam") + else: + log.info("Refreshing clamav db...") + try: + redo.retry(lambda: sh.freshclam("--stdout", "--verbose", + _timeout=300, _err_to_out=True)) + log.info("Done.") + except sh.ErrorReturnCode: + log.warning("Freshclam failed, skipping DB update") + manifest = [] + for e in task["extra"]["funsize"]["partials"]: + for mar in (e["from_mar"], e["to_mar"]): + verify_allowed_url(mar) + + work_env = WorkEnv() + # TODO: run setup once + work_env.setup() + complete_mars = {} + for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])): + dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type)) + unpack_dir = os.path.join(work_env.workdir, mar_type) + download(f, dest) + if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"): + verify_signature(dest, args.signing_cert) + complete_mars["%s_size" % mar_type] = os.path.getsize(dest) + complete_mars["%s_hash" % mar_type] = get_hash(dest) + unpack(work_env, dest, unpack_dir) + log.info("AV-scanning %s ...", unpack_dir) + sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True) + log.info("Done.") + + path = os.path.join(work_env.workdir, "to") + from_path = os.path.join(work_env.workdir, "from") + mar_data = { + "ACCEPTED_MAR_CHANNEL_IDS": get_option( + path, filename="update-settings.ini", section="Settings", + option="ACCEPTED_MAR_CHANNEL_IDS"), + "version": get_option(path, filename="application.ini", + section="App", option="Version"), + "to_buildid": get_option(path, filename="application.ini", + section="App", option="BuildID"), + "from_buildid": get_option(from_path, filename="application.ini", + section="App", option="BuildID"), + "appName": get_option(from_path, filename="application.ini", + section="App", option="Name"), + # Use Gecko repo and rev from platform.ini, not application.ini + "repo": get_option(path, filename="platform.ini", section="Build", + option="SourceRepository"), + "revision": get_option(path, filename="platform.ini", + section="Build", option="SourceStamp"), + "from_mar": e["from_mar"], + "to_mar": e["to_mar"], + "platform": e["platform"], + "locale": e["locale"], + } + # Override ACCEPTED_MAR_CHANNEL_IDS if needed + if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ: + mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ["ACCEPTED_MAR_CHANNEL_IDS"] + for field in ("update_number", "previousVersion", + "previousBuildNumber", "toVersion", + "toBuildNumber"): + if field in e: + mar_data[field] = e[field] + mar_data.update(complete_mars) + # if branch not set explicitly use repo-name + mar_data["branch"] = e.get("branch", + mar_data["repo"].rstrip("/").split("/")[-1]) + mar_name = args.filename_template.format(**mar_data) + mar_data["mar"] = mar_name + dest_mar = os.path.join(work_env.workdir, mar_name) + # TODO: download these once + work_env.download_buildsystem_bits(repo=mar_data["repo"], + revision=mar_data["revision"]) + generate_partial(work_env, from_path, path, dest_mar, + mar_data["ACCEPTED_MAR_CHANNEL_IDS"], + mar_data["version"]) + mar_data["size"] = os.path.getsize(dest_mar) + mar_data["hash"] = get_hash(dest_mar) + + shutil.copy(dest_mar, args.artifacts_dir) + work_env.cleanup() + manifest.append(mar_data) + manifest_file = os.path.join(args.artifacts_dir, "manifest.json") + with open(manifest_file, "w") as fp: + json.dump(manifest, fp, indent=2, sort_keys=True) + + +if __name__ == '__main__': + main() diff -Nru firefox-52.0+build2/testing/docker/funsize-update-generator/scripts/mbsdiff_hook.sh firefox-52.0.1+build2/testing/docker/funsize-update-generator/scripts/mbsdiff_hook.sh --- firefox-52.0+build2/testing/docker/funsize-update-generator/scripts/mbsdiff_hook.sh 1970-01-01 00:00:00.000000000 +0000 +++ firefox-52.0.1+build2/testing/docker/funsize-update-generator/scripts/mbsdiff_hook.sh 2017-03-17 13:41:06.000000000 +0000 @@ -0,0 +1,135 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool contains functions that are to be used to handle/enable funsize +# Author: Mihai Tabara +# + +HOOK= +SERVER_URL= +LOCAL_CACHE_DIR= + +getsha512(){ + echo "$(openssl sha512 "${1}" | awk '{print $2}')" +} + +print_usage(){ + echo "$(basename $0) -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] PATH-FROM-URL PATH-TO-URL PATH-PATCH" + echo "Script that saves/retrieves from cache presumptive patches as args" + echo "" + echo "-A SERVER-URL - host where to send the files" + echo "-c LOCAL-CACHE-DIR-PATH local path to which patches are cached" + echo "-g pre hook - tests whether patch already in cache" + echo "-u post hook - upload patch to cache for future use" + echo "" + echo "PATH-FROM-URL : path on disk for source file" + echo "PATH-TO-URL : path on disk for destination file" + echo "PATH-PATCH : path on disk for patch between source and destination" +} + +upload_patch(){ + sha_from=`getsha512 "$1"` + sha_to=`getsha512 "$2"` + patch_path="$3" + + # save to local cache first + if [ -n "$LOCAL_CACHE_DIR" ]; then + local_cmd="mkdir -p "$LOCAL_CACHE_DIR/$sha_from"" + if `$local_cmd` >&2; then + cp -avf "$patch_path" "$LOCAL_CACHE_DIR/$sha_from/$sha_to" + echo "$patch_path saved on local cache!" + fi + fi + # The remote cache implementation is not used. The code is for usage + # reference only. + return 0 + + # send it over to funsize + cmd="curl -sSw %{http_code} -o /dev/null -X POST $SERVER_URL -F sha_from="$sha_from" -F sha_to="$sha_to" -F patch_file="@$patch_path"" + ret_code=`$cmd` + + if [ $ret_code -eq 200 ]; then + echo "$patch_path Successful uploaded to funsize!" + return 0 + fi + + echo "$patch_path Failed to be uploaded to funsize!" + return 1 +} + +get_patch(){ + sha_from=`getsha512 "$1"` + sha_to=`getsha512 "$2"` + destination_file="$3" + tmp_file="$destination_file.tmp" + + # try to retrieve from local cache first + if [ -r "$LOCAL_CACHE_DIR/$sha_from/$sha_to" ]; then + cp -avf "$LOCAL_CACHE_DIR/$sha_from/$sha_to" "$destination_file" + echo "Successful retrieved $destination_file from local cache!" + return 0 + else + echo "File is not in the locale cache" + return 1 + fi + # The remote cache implementation is not used. The code is for usage + # reference only. + + # if unsuccessful, try to retrieve from funsize + cmd="curl -LsSGw %{http_code} $SERVER_URL/$sha_from/$sha_to -o $tmp_file" + ret_code=`$cmd` + + if [ $ret_code -eq 200 ]; then + mv "$tmp_file" "$destination_file" + echo "Successful retrieved $destination_file from funsize!" + return 0 + fi + + rm -f "$tmp_file" + echo "Failed to retrieve $destination_file from funsize!" + return 1 +} + +OPTIND=1 + +while getopts ":A:c:gu" option; do + case $option in + A) + SERVER_URL="$OPTARG" + ;; + c) + LOCAL_CACHE_DIR="$OPTARG" + ;; + g) + HOOK="PRE" + ;; + u) + HOOK="POST" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + print_usage + exit 1 + ;; + *) + echo "Unimplemented option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ "$HOOK" == "PRE" ]; then + get_patch "$1" "$2" "$3" +elif [ "$HOOK" == "POST" ]; then + upload_patch "$1" "$2" "$3" +fi diff -Nru firefox-52.0+build2/testing/mozharness/configs/releases/bouncer_firefox_beta.py firefox-52.0.1+build2/testing/mozharness/configs/releases/bouncer_firefox_beta.py --- firefox-52.0+build2/testing/mozharness/configs/releases/bouncer_firefox_beta.py 2017-03-02 23:36:37.000000000 +0000 +++ firefox-52.0.1+build2/testing/mozharness/configs/releases/bouncer_firefox_beta.py 2017-03-17 13:41:07.000000000 +0000 @@ -70,6 +70,10 @@ "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe", "bouncer-platform": "win", }, + "win64": { + "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe", + "bouncer-platform": "win64", + }, }, }, "sha1-installer": { diff -Nru firefox-52.0+build2/testing/mozharness/configs/releases/bouncer_firefox_release.py firefox-52.0.1+build2/testing/mozharness/configs/releases/bouncer_firefox_release.py --- firefox-52.0+build2/testing/mozharness/configs/releases/bouncer_firefox_release.py 2017-03-02 23:36:37.000000000 +0000 +++ firefox-52.0.1+build2/testing/mozharness/configs/releases/bouncer_firefox_release.py 2017-03-17 13:41:07.000000000 +0000 @@ -70,6 +70,10 @@ "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe", "bouncer-platform": "win", }, + "win64": { + "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe", + "bouncer-platform": "win64", + }, }, }, "complete-mar": { diff -Nru firefox-52.0+build2/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py firefox-52.0.1+build2/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py --- firefox-52.0+build2/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py 2017-03-02 23:36:37.000000000 +0000 +++ firefox-52.0.1+build2/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py 2017-03-17 13:41:07.000000000 +0000 @@ -69,6 +69,10 @@ "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe", "bouncer-platform": "win", }, + "win64": { + "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe", + "bouncer-platform": "win64", + }, }, }, "complete-mar": { diff -Nru firefox-52.0+build2/toolkit/mozapps/installer/packager.mk firefox-52.0.1+build2/toolkit/mozapps/installer/packager.mk --- firefox-52.0+build2/toolkit/mozapps/installer/packager.mk 2017-03-02 23:36:40.000000000 +0000 +++ firefox-52.0.1+build2/toolkit/mozapps/installer/packager.mk 2017-03-17 13:41:11.000000000 +0000 @@ -100,7 +100,8 @@ GARBAGE += make-package make-sourcestamp-file:: - @awk '$$2 == "MOZ_BUILDID" {print $$3}' $(DEPTH)/buildid.h > $(MOZ_SOURCESTAMP_FILE) + $(NSINSTALL) -D $(DIST)/$(PKG_PATH) + @echo '$(BUILDID)' > $(MOZ_SOURCESTAMP_FILE) ifdef MOZ_INCLUDE_SOURCE_INFO @awk '$$2 == "MOZ_SOURCE_URL" {print $$3}' $(DEPTH)/source-repo.h >> $(MOZ_SOURCESTAMP_FILE) endif @@ -230,7 +231,7 @@ @echo 'Generate the sourcestamp file' # Make sure to have repository information available and then generate the # sourcestamp file. - $(MAKE) -C $(DEPTH) 'source-repo.h' 'buildid.h' + $(MAKE) -C $(DEPTH) 'source-repo.h' $(MAKE) make-sourcestamp-file @echo 'Packaging source tarball...' # We want to include the sourcestamp file in the source tarball, so copy it