diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/bin/test_buildd_generatetranslationtemplates launchpad-buildd-237~660~ubuntu20.04.1/bin/test_buildd_generatetranslationtemplates --- launchpad-buildd-236~650~ubuntu20.04.1/bin/test_buildd_generatetranslationtemplates 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/bin/test_buildd_generatetranslationtemplates 2024-04-30 12:51:39.000000000 +0000 @@ -19,6 +19,7 @@ proxy = ServerProxy("http://localhost:8221/rpc") print(proxy.info()) +print(proxy.proxy_info()) print(proxy.status()) buildid = "1-2" build_type = "translation-templates" diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/bin/test_buildd_recipe launchpad-buildd-237~660~ubuntu20.04.1/bin/test_buildd_recipe --- launchpad-buildd-236~650~ubuntu20.04.1/bin/test_buildd_recipe 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/bin/test_buildd_recipe 2024-04-30 12:51:39.000000000 +0000 @@ -25,6 +25,7 @@ proxy = ServerProxy("http://localhost:8221/rpc") print(proxy.echo("Hello World")) print(proxy.info()) +print(proxy.proxy_info()) status = proxy.status() print(status) if status[0] != "BuilderStatus.IDLE": diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/debian/changelog launchpad-buildd-237~660~ubuntu20.04.1/debian/changelog --- launchpad-buildd-236~650~ubuntu20.04.1/debian/changelog 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/debian/changelog 2024-04-30 12:51:39.000000000 +0000 @@ -1,8 +1,24 @@ -launchpad-buildd (236~650~ubuntu20.04.1) focal; urgency=low +launchpad-buildd (237~660~ubuntu20.04.1) focal; urgency=low * Auto build. - -- Launchpad Package Builder Mon, 19 Feb 2024 14:02:58 +0000 + -- Ines Almeida Tue, 30 Apr 2024 12:51:39 +0000 + +launchpad-buildd (237) focal; urgency=medium + + [ Simone Pelosi ] + * Improve documentation for qastaging deployment. + + [ Inês Almeida ] + * Add logic to allow using fetch-service as the builder proxy for snaps that + have the `use_fetch_service` flag on. + Update token revocation authentication when using the fetch service. + Install mitm-certificates that are now injected from the buildd-manager. + * Improve documentation for qastaging deployment. + * Add `proxy_info()` xmlrpc endpoint that can be used to retrieve proxy + details from a builder. + + -- Inês Almeida Wed, 24 Apr 2024 13:20:40 +0200 launchpad-buildd (236) focal; urgency=medium diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/debian/git-build-recipe.manifest launchpad-buildd-237~660~ubuntu20.04.1/debian/git-build-recipe.manifest --- launchpad-buildd-236~650~ubuntu20.04.1/debian/git-build-recipe.manifest 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/debian/git-build-recipe.manifest 2024-04-30 12:51:39.000000000 +0000 @@ -1,2 +1,2 @@ -# git-build-recipe format 0.4 deb-version {debupstream}~650 -lp:launchpad-buildd git-commit:955119cb2ce86b93e1b73d877463a6a44e2941e5 +# git-build-recipe format 0.4 deb-version {debupstream}~660 +lp:launchpad-buildd git-commit:53faeabb12999bcde9deac944a04461c8e0ba44b diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/docs/how-to/deployment.rst launchpad-buildd-237~660~ubuntu20.04.1/docs/how-to/deployment.rst --- launchpad-buildd-236~650~ubuntu20.04.1/docs/how-to/deployment.rst 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/docs/how-to/deployment.rst 2024-04-30 12:51:39.000000000 +0000 @@ -36,13 +36,16 @@ --to=ppa:launchpad/ubuntu/buildd-staging -b launchpad-buildd`` (from ``ubuntu-archive-tools``) to copy the current version of launchpad-buildd to the deployment PPA (``jammy`` here refers to the series being used on - qastaging builder instances). + the builder instances). + + For example, in qastaging, we have builders in ``focal`` and in ``jammy``, + so you should run the command for both series. #. `Wait for PPA publishing to complete `__. #. Run ``mojo run -m manifest-rebuild-images`` in the management environment - (``stg-vbuilder@launchpad-bastion-ps5``) to start rebuilding images. + (``stg-vbuilder-qastaging@launchpad-bastion-ps5``) to start rebuilding images. After a minute or so, ``juju status glance-simplestreams-sync-\*`` will show "Synchronising images"; once this says "Sync completed", images have been rebuilt. diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/builder.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/builder.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/builder.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/builder.py 2024-04-30 12:51:39.000000000 +0000 @@ -786,6 +786,11 @@ """Echo the argument back.""" return args + def xmlrpc_proxy_info(self): + """Return the details for the proxy used by the manager.""" + proxy_fields = ["use_fetch_service", "revocation_endpoint"] + return {k: getattr(self.builder.manager, k) for k in proxy_fields} + def xmlrpc_info(self): """Return the protocol version and the manager methods supported.""" return ( diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/proxy.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/proxy.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/proxy.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/proxy.py 2024-04-30 12:51:39.000000000 +0000 @@ -214,8 +214,20 @@ class BuildManagerProxyMixin: + @property + def _use_fetch_service(self): + return hasattr(self, "use_fetch_service") and getattr( + self, "use_fetch_service" + ) + def startProxy(self): - """Start the local builder proxy, if necessary.""" + """Start the local builder proxy, if necessary. + + This starts an internal proxy that stands before the Builder + Proxy/Fetch Service, so build systems, which do not comply + with standard `http(s)_proxy` environment variables, would + still work with the builder proxy. + """ if not self.proxy_url: return [] proxy_port = self._builder._config.get("builder", "proxyport") @@ -231,7 +243,7 @@ return ["--proxy-url", f"http://{proxy_host}:{proxy_port}/"] def stopProxy(self): - """Stop the local builder proxy, if necessary.""" + """Stop the internal local proxy (see `startProxy`), if necessary.""" if self.proxy_service is None: return self.proxy_service.disownServiceParent() @@ -243,6 +255,10 @@ return self._builder.log("Revoking proxy token...\n") try: - revoke_proxy_token(self.proxy_url, self.revocation_endpoint) + revoke_proxy_token( + self.proxy_url, + self.revocation_endpoint, + self._use_fetch_service, + ) except RevokeProxyTokenError as e: self._builder.log(f"{e}\n") diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/snap.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/snap.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/snap.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/snap.py 2024-04-30 12:51:39.000000000 +0000 @@ -38,7 +38,10 @@ self.branch = extra_args.get("branch") self.git_repository = extra_args.get("git_repository") self.git_path = extra_args.get("git_path") + self.use_fetch_service = extra_args.get("use_fetch_service") self.proxy_url = extra_args.get("proxy_url") + # currently only used to transport the mitm certificate + self.secrets = extra_args.get("secrets") self.revocation_endpoint = extra_args.get("revocation_endpoint") self.build_source_tarball = extra_args.get( "build_source_tarball", False @@ -100,6 +103,17 @@ if self.target_architectures: for arch in self.target_architectures: args.extend(["--target-arch", arch]) + if self.use_fetch_service: + args.append("--use_fetch_service") + # XXX 2024-04-17 jugmac00: I do not think we need to add checks + # whether this information is present, as otherwise the fetch + # service won't work anyway + args.extend( + [ + "--fetch-service-mitm-certificate", + self.secrets["fetch_service_mitm_certificate"], + ] + ) args.append(self.name) self.runTargetSubProcess("buildsnap", *args) diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/target/build_snap.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/target/build_snap.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/target/build_snap.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/target/build_snap.py 2024-04-30 12:51:39.000000000 +0000 @@ -2,6 +2,7 @@ # GNU Affero General Public License version 3 (see the file LICENSE). import argparse +import base64 import json import logging import os.path @@ -102,8 +103,41 @@ action="store_true", help="disable proxy access after the pull phase has finished", ) + parser.add_argument( + "--use_fetch_service", + default=False, + action="store_true", + help="use the fetch service instead of the builder proxy", + ) + parser.add_argument( + "--fetch-service-mitm-certificate", + type=str, + help="content of the ca certificate", + ) parser.add_argument("name", help="name of snap to build") + def install_mitm_certificate(self): + """Install ca certificate for the fetch service + + This is necessary so the fetch service can man-in-the-middle all + requests when fetching dependencies. + """ + with self.backend.open( + "/usr/local/share/ca-certificates/local-ca.crt", + mode="wb" + ) as local_ca_cert: + # Certificate is passed as a Base64 encoded string. + # It's encoded using `base64 -w0` on the cert file. + decoded_certificate = base64.b64decode( + self.args.fetch_service_mitm_certificate.encode("ASCII") + ) + local_ca_cert.write(decoded_certificate) + os.fchmod(local_ca_cert.fileno(), 0o644) + self.backend.run(["update-ca-certificates"]) + # XXX jugmac00 2024-04-17: We might need to restart snapd + # so the new certificate will be used + # snapd folks are unsure, so we need to try ourselves + def install_svn_servers(self): proxy = urlparse(self.args.proxy_url) svn_servers = dedent( @@ -172,7 +206,13 @@ ] ) if self.args.proxy_url: + # XXX jugmac00 2024-04-17: this is configuring an SVN server; + # it is currently unclear whether this is still necessary for + # building snaps + # jugmac00 reached out both to William and Claudio to figure out self.install_svn_servers() + if self.args.use_fetch_service: + self.install_mitm_certificate() def repo(self): """Collect git or bzr branch.""" @@ -231,7 +271,9 @@ logger.info("Revoking proxy token...") try: revoke_proxy_token( - self.args.upstream_proxy_url, self.args.revocation_endpoint + self.args.upstream_proxy_url, + self.args.revocation_endpoint, + self.args.use_fetch_service, ) except RevokeProxyTokenError as e: logger.info(str(e)) diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/target/tests/test_build_snap.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/target/tests/test_build_snap.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/target/tests/test_build_snap.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/target/tests/test_build_snap.py 2024-04-30 12:51:39.000000000 +0000 @@ -194,6 +194,66 @@ build_snap.backend.backend_fs["/root/.subversion/servers"], ) + def test_install_certificate(self): + args = [ + "buildsnap", + "--backend=fake", + "--series=xenial", + "--arch=amd64", + "1", + "--git-repository", + "lp:foo", + "--proxy-url", + "http://proxy.example:3128/", + "test-snap", + "--use_fetch_service", + "--fetch-service-mitm-certificate", + # Base64 content_of_cert + "Y29udGVudF9vZl9jZXJ0", + ] + build_snap = parse_args(args=args).operation + build_snap.bin = "/builderbin" + self.useFixture(FakeFilesystem()).add("/builderbin") + os.mkdir("/builderbin") + with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script: + proxy_script.write("proxy script\n") + os.fchmod(proxy_script.fileno(), 0o755) + build_snap.install() + self.assertThat( + build_snap.backend.run.calls, + MatchesListwise( + [ + RanAptGet( + "install", "python3", "socat", "git", "snapcraft" + ), + RanCommand(["mkdir", "-p", "/root/.subversion"]), + RanCommand(["update-ca-certificates"]), + ] + ), + ) + self.assertEqual( + (b"proxy script\n", stat.S_IFREG | 0o755), + build_snap.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"], + ) + self.assertEqual( + ( + b"[global]\n" + b"http-proxy-host = proxy.example\n" + b"http-proxy-port = 3128\n", + stat.S_IFREG | 0o644, + ), + build_snap.backend.backend_fs["/root/.subversion/servers"], + ) + self.assertEqual( + ( + b"content_of_cert", + stat.S_IFREG | 0o644, + ), + build_snap.backend.backend_fs[ + "/usr/local/share/ca-certificates/local-ca.crt" + ], + ) + def test_install_channels(self): args = [ "buildsnap", diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/tests/test_snap.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/tests/test_snap.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/tests/test_snap.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/tests/test_snap.py 2024-04-30 12:51:39.000000000 +0000 @@ -283,7 +283,8 @@ "/build/test-snap/test-snap_0_all.snap", b"I am a snap package." ) self.buildmanager.backend.add_file( - "/build/test-snap/test-snap+somecomponent_0.comp", b"I am a component." + "/build/test-snap/test-snap+somecomponent_0.comp", + b"I am a component.", ) # After building the package, reap processes. @@ -754,6 +755,21 @@ yield self.startBuild(args, expected_options) @defer.inlineCallbacks + def test_iterate_use_fetch_service(self): + # The build manager can be told to use the fetch service as its proxy. + # This requires also a ca certificate passed in via secrets. + args = { + "use_fetch_service": True, + "secrets": {"fetch_service_mitm_certificate": "content_of_cert"}, + } + expected_options = [ + "--use_fetch_service", + "--fetch-service-mitm-certificate", + "content_of_cert", + ] + yield self.startBuild(args, expected_options) + + @defer.inlineCallbacks def test_iterate_disable_proxy_after_pull(self): self.builder._config.set("builder", "proxyport", "8222") args = { @@ -874,3 +890,29 @@ # XXX cjwatson 2023-02-07: Ideally we'd check the timeout as well, # but the version of responses in Ubuntu 20.04 doesn't store it # anywhere we can get at it. + + @responses.activate + def test_revokeProxyToken_fetch_service(self): + session_id = "123" + + responses.add( + "DELETE", + f"http://control.fetch-service.example/{session_id}/token", + ) + + self.buildmanager.use_fetch_service = True + self.buildmanager.revocation_endpoint = ( + f"http://control.fetch-service.example/{session_id}/token" + ) + self.buildmanager.proxy_url = ( + "http://session_id:token@proxy.fetch-service.example" + ) + + self.buildmanager.revokeProxyToken() + + self.assertEqual(1, len(responses.calls)) + request = responses.calls[0].request + self.assertEqual( + f"http://control.fetch-service.example/{session_id}/token", + request.url, + ) diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/tests/test_util.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/tests/test_util.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/tests/test_util.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/tests/test_util.py 2024-04-30 12:51:39.000000000 +0000 @@ -1,9 +1,17 @@ # Copyright 2017 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). +import base64 + +import responses from testtools import TestCase -from lpbuildd.util import get_arch_bits, set_personality, shell_escape +from lpbuildd.util import ( + get_arch_bits, + revoke_proxy_token, + set_personality, + shell_escape, +) class TestShellEscape(TestCase): @@ -59,3 +67,49 @@ ["linux64", "sbuild"], set_personality(["sbuild"], "amd64", series="trusty"), ) + + +class TestRevokeToken(TestCase): + @responses.activate + def test_revoke_proxy_token(self): + """Proxy token revocation uses the right authentication""" + + proxy_url = "http://username:password@proxy.example" + revocation_endpoint = "http://proxy-auth.example/tokens/build_id" + token = base64.b64encode(b"username:password").decode() + + responses.add(responses.DELETE, revocation_endpoint) + + revoke_proxy_token(proxy_url, revocation_endpoint) + self.assertEqual(1, len(responses.calls)) + request = responses.calls[0].request + self.assertEqual( + "http://proxy-auth.example/tokens/build_id", request.url + ) + self.assertEqual(f"Basic {token}", request.headers["Authorization"]) + + @responses.activate + def test_revoke_fetch_service_token(self): + """Proxy token revocation for the fetch service""" + + token = "token" + proxy_url = f"http://session_id:{token}@proxy.fetch-service.example" + revocation_endpoint = ( + "http://control.fetch-service.example/session_id/token" + ) + + responses.add(responses.DELETE, revocation_endpoint) + + revoke_proxy_token( + proxy_url, + revocation_endpoint, + use_fetch_service=True, + ) + + self.assertEqual(1, len(responses.calls)) + request = responses.calls[0].request + self.assertEqual( + "http://control.fetch-service.example/session_id/token", + request.url, + ) + self.assertEqual(f"Basic {token}", request.headers["Authorization"]) diff -Nru launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/util.py launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/util.py --- launchpad-buildd-236~650~ubuntu20.04.1/lpbuildd/util.py 2024-02-19 14:02:58.000000000 +0000 +++ launchpad-buildd-237~660~ubuntu20.04.1/lpbuildd/util.py 2024-04-30 12:51:39.000000000 +0000 @@ -1,6 +1,7 @@ # Copyright 2015-2017 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). +import base64 import os import subprocess import sys @@ -68,15 +69,38 @@ return f"Unable to revoke token for {self.username}: {self.exception}" -def revoke_proxy_token(proxy_url, revocation_endpoint): +def revoke_proxy_token( + proxy_url, revocation_endpoint, use_fetch_service=False +): """Revoke builder proxy token. + If not using the fetch service: + The proxy_url for the current Builder Proxy has the following format: + http://{username}:{password}@{host}:{port} + + We use the username-password combo from the proxy_url for + authentication to revoke its token. + + If using the fetch service: + The proxy_url for the Fetch Service has the following format: + http://{session_id}:{token}@{host}:{port} + + We use the token from the proxy_url for authentication to revoke + elself. + :raises RevokeProxyTokenError: if attempting to revoke the token failed. """ url = urlparse(proxy_url) + + if not use_fetch_service: + auth_string = f"{url.username}:{url.password}" + token = base64.b64encode(auth_string.encode()).decode() + else: + token = url.password + + headers = {"Authorization": f"Basic {token}"} + try: - requests.delete( - revocation_endpoint, auth=(url.username, url.password), timeout=15 - ) + requests.delete(revocation_endpoint, headers=headers, timeout=15) except requests.RequestException as e: raise RevokeProxyTokenError(url.username, e)