diff -Nru maas-1.5+bzr2252/CHANGELOG maas-1.5.4+bzr2294/CHANGELOG --- maas-1.5+bzr2252/CHANGELOG 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/CHANGELOG 2014-09-03 14:18:31.000000000 +0000 @@ -2,6 +2,79 @@ Changelog ========= +1.5.4 +===== + +Bug fix update +-------------- + + - Package fails to install when the default route is through an + aliased/tagged interface (LP: #1350235) + - ERROR Nonce already used (LP: #1190986) + - Add MAAS arm64/xgene support (LP: #1338851) + - Add utopic support (LP: #1337437) + - API documentation for nodegroup op=details missing parameter + (LP: #1331982) + + +1.5.3 +===== + +Bug fix update +-------------- + + - Reduce number of celery tasks emitted when updating a cluster controller + (LP: #1324944) + - Fix VirshSSH template which was referencing invalid attributes + (LP: #1324966) + - Fix a start up problem where a database lock was being taken outside of + a transaction (LP: #1325759) + - Reformat badly formatted Architecture error message (LP: #1301465) + - Final changes to support ppc64el (now known as PowerNV) (LP: #1315154) + + +1.5.2 +===== + +Bug fix update +-------------- + +- Remove workaround for fixed Django bug 1311433 (LP: #1311433) +- Ensure that validation errors are returned when adding a node over + the API and its cluster controller is not contactable. (LP: #1305061) +- Hardware enablement support for PowerKVM +- Shorten the time taken for a cluster to initially connect to the region + via RPC to around 2 seconds (LP: #1317682) +- Faster DHCP leases parser (LP: #1305102) +- Documentation fixed explaining how to enable an ephemeral backdoor + (LP: #1321696) +- Use probe-and-enlist-hardware to enlist all virtual machine inside + a libvirt machine, allow password qemu+ssh connections. + (LP: #1315155, LP: #1315157) +- Rename ppc64el boot loader to PowerKVM (LP: #1315154) + + +1.5.1 +===== + +Bug fix update +-------------- + +For full details see https://launchpad.net/maas/+milestone/1.5.1 + +#1303915 Powering SM15k RESTAPI v2.0 doesn't force PXE boot +#1307780 no armhf commissioning template +#1310076 lost connectivity to a node when using fastpath-installer with precise+hwe-s +#1310082 d-i with precise+hwe-s stops at "Architecture not supported" +#1311151 MAAS imports Trusty's 'rc' images by default. +#1311433 REGRESSION: AttributeError: 'functools.partial' object has no attribute '__module__' +#1313556 API client blocks when deleting a resource +#1314409 parallel juju deployments race on the same maas +#1316396 When stopping a node from the web UI that was started from the API, distro_series is not cleared +#1298784 Vulnerable to user-interface redressing (e.g. clickjacking) +#1308772 maas has no way to specify alternate IP addresses for AMT template +#1300476 Unable to setup BMC/UCS user on Cisco B200 M3 + 1.5 === diff -Nru maas-1.5+bzr2252/contrib/preseeds_v2/curtin_userdata maas-1.5.4+bzr2294/contrib/preseeds_v2/curtin_userdata --- maas-1.5+bzr2252/contrib/preseeds_v2/curtin_userdata 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/contrib/preseeds_v2/curtin_userdata 2014-09-03 14:18:31.000000000 +0000 @@ -28,7 +28,7 @@ power_state: mode: reboot -{{if node.architecture in {'i386/generic', 'amd64/generic'} }} +{{if node.split_arch()[0] in {'i386', 'amd64'} }} apt_mirrors: ubuntu_archive: http://{{main_archive_hostname}}/{{main_archive_directory}} ubuntu_security: http://{{main_archive_hostname}}/{{main_archive_directory}} diff -Nru maas-1.5+bzr2252/contrib/preseeds_v2/preseed_master maas-1.5.4+bzr2294/contrib/preseeds_v2/preseed_master --- maas-1.5+bzr2252/contrib/preseeds_v2/preseed_master 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/contrib/preseeds_v2/preseed_master 2014-09-03 14:18:31.000000000 +0000 @@ -40,8 +40,10 @@ d-i partman/confirm_nooverwrite boolean true d-i partman/default_filesystem string ext4 -# Use server kernel -d-i base-installer/kernel/image string linux-server +# Enable this if you want to override to a specific kernel, such as +# linux-generic-lts-saucy, but Debian Installer should pick the right one based +# on the boot kernel. +#d-i base-installer/kernel/image string linux-server # User Setup d-i passwd/root-login boolean false diff -Nru maas-1.5+bzr2252/debian/changelog maas-1.5.4+bzr2294/debian/changelog --- maas-1.5+bzr2252/debian/changelog 2014-04-15 16:07:37.000000000 +0000 +++ maas-1.5.4+bzr2294/debian/changelog 2014-12-04 18:59:56.000000000 +0000 @@ -1,3 +1,117 @@ +maas (1.5.4+bzr2294-0ubuntu1.2) trusty-security; urgency=medium + + * Fix compatibility with mod-wsgi security update (LP: #1399016) + - debian/patches/home-directory.patch: specify a valid home directory + for the maas user, since mod-wsgi no longer works without one. + + -- Marc Deslauriers Thu, 04 Dec 2014 13:59:56 -0500 + +maas (1.5.4+bzr2294-0ubuntu1.1) trusty-proposed; urgency=medium + + * Add hardware enablement for armhf/keystone (LP: #1350103) + + -- Greg Lutostanski Thu, 18 Sep 2014 16:43:56 -0500 + +maas (1.5.4+bzr2294-0ubuntu1) trusty-proposed; urgency=medium + + * New upstream bug fix release: + - Change supported releases for install to Precise, Saucy, Trusty, Utopic + (Add Utopic; Remove Quantal/Raring) -- will still only be able to install + releases with streams available to maas (LP: #1337437) + - Package fails to install when the default route is through an + aliased/tagged interface (LP: #1350235) + - ERROR Nonce already used (LP: #1190986) + - Add MAAS arm64/xgene support (LP: #1338851) + - API documentation for nodegroup op=details missing parameter + (LP: #1331982) + - Reduce number of celery tasks emitted when updating a cluster controller + (LP: #1324944) + - Fix VirshSSH template which was referencing invalid attributes + (LP: #1324966) + - Fix a start up problems where a database lock was being taken outside of + a transaction (LP: #1325640, LP: #1325759) + - Reformat badly formatted Architecture error message (LP: #1301465) + - Final changes to support ppc64el (now known as PowerNV) (LP: #1315154) + - UI tweak to make navigation elements visible for documentation + + [ Greg Lutostanski ] + * debian/control: + - maas-provisioningserver not maas-cluster-controller depends on + python-pexpect (LP: #1352273) + + [ Gavin Panella ] + * debian/maas-cluster-controller.postinst + - Allow maas-pserv to bind to all IPv6 addresses too. (LP: #1342302) + + [ Diogo Matsubara ] + * debian/control: + - python-maas-provisioningserver depends on python-paramiko (LP: #1334401) + + [ Raphaël Badin ] + * debian/extras/99-maas-sudoers: + - Add rule 'maas-dhcp-server stop' job. + + -- Greg Lutostanski Fri, 29 Aug 2014 13:27:34 -0500 + +maas (1.5.2+bzr2282-0ubuntu0.2) trusty-proposed; urgency=medium + + * debian/control: + - Add missing dependency in maas-cluster-controller for grub-common + (LP: #1328231) + - Move dependency from maas-cluster-controller to maas-provisioningserver + for python-seamicroclient (LP: #1332532) + + -- Greg Lutostanski Fri, 20 Jun 2014 10:10:47 -0500 + +maas (1.5.2+bzr2282-0ubuntu0.1) trusty-proposed; urgency=medium + + * New upstream release: + - Remove workaround for fixed Django bug 1311433 (LP: #1311433) + - Ensure that validation errors are returned when adding a node over + the API and its cluster controller is not contactable. (LP: #1305061) + - Hardware enablement support for PowerKVM (LP: #1325038) + - Shorten the time taken for a cluster to initially connect to the region + via RPC to around 2 seconds (LP: #1317682) + - Faster DHCP leases parser (LP: #1305102) + - Documentation fixed explaining how to enable an ephemeral backdoor + (LP: #1321696) + - Use probe-and-enlist-hardware to enlist all virtual machine inside + a libvirt machine, allow password qemu+ssh connections. + (LP: #1315155, LP: #1315157) + - Rename ppc64el boot loader to PowerKVM (LP: #1315154) + - Fix NodeForm's is_valid() method so that it uses Django's way of setting + errors on forms instead of putting text in self.errors['architecture'] + (LP: #1301465) + - Change BootMethods to return their own IReader per-request, update method + names to reflect new usage. (LP: #1315154) + - Return early and stop the DHCP server when the list of managed interfaces + of the nodegroup is empty. (LP: #1324944) + - Fix invalid attribute references in the VirshSSH class. Added more test + for the VirshSSH class. (LP: #1324966) + * debian/control: + - Add missing dependency in maas-cluster-controller for python-pexpect + (LP: #1322151) + + -- Greg Lutostanski Wed, 04 Jun 2014 14:31:41 -0500 + +maas (1.5.1+bzr2269-0ubuntu0.1) trusty; urgency=medium + + * Stable Release Update (LP: #1317601): + - Hardware Enablement for Cisco B-Series. (LP: #1300476) + - Allow AMT power type to specify IP Address. (LP: #1308772) + - Spurious failure when starting and creating lock files. (LP: 1308069) + - Fix usage of hardware enablement kernels by fixing the preseeds + (LP: #1310082, LP: #1310076, LP: #1310082) + - Fix parallel juju deployments. (LP: #1314409) + - Clear distro_series when stopping node from WebUI (LP: #1316396) + - Fix click hijacking (LP: #1298784) + - Fix blocking API client when deleting a resource (LP: #1313556) + - Do not import Trusty RC images by default (LP: #1311151) + - debian/control: Add missing dep on python-crochet for + python-maas-provisioningserver (LP: #1311765) + + -- Andres Rodriguez Fri, 09 May 2014 22:35:43 -0500 + maas (1.5+bzr2252-0ubuntu1) trusty; urgency=medium * New upstream release diff -Nru maas-1.5+bzr2252/debian/control maas-1.5.4+bzr2294/debian/control --- maas-1.5+bzr2252/debian/control 2014-04-15 16:06:26.000000000 +0000 +++ maas-1.5.4+bzr2294/debian/control 2014-09-19 20:32:28.000000000 +0000 @@ -87,6 +87,7 @@ Architecture: all Depends: python-amqp, python-celery, + python-crochet, python-distro-info, python-formencode, python-jsonschema, @@ -98,7 +99,10 @@ python-oops-amqp, python-oops-datedir-repo, python-oops-twisted, + python-paramiko, + python-pexpect, python-pyparsing, + python-seamicroclient, python-simplestreams, python-tempita, python-twisted-core, @@ -146,6 +150,7 @@ bind9utils, distro-info, freeipmi-tools, + grub-common, maas-cli (=${binary:Version}), maas-common (=${binary:Version}), maas-dhcp (=${binary:Version}), @@ -156,7 +161,6 @@ python-maas-provisioningserver (=${binary:Version}), python-netaddr, python-oauth, - python-seamicroclient, python-tempita, python-twisted, python-zope.interface, diff -Nru maas-1.5+bzr2252/debian/extras/99-maas-sudoers maas-1.5.4+bzr2294/debian/extras/99-maas-sudoers --- maas-1.5+bzr2252/debian/extras/99-maas-sudoers 2014-04-15 16:06:26.000000000 +0000 +++ maas-1.5.4+bzr2294/debian/extras/99-maas-sudoers 2014-09-19 20:32:28.000000000 +0000 @@ -1,3 +1,4 @@ maas ALL= NOPASSWD: /usr/sbin/service maas-dhcp-server restart +maas ALL= NOPASSWD: /usr/sbin/service maas-dhcp-server stop maas ALL= NOPASSWD: /usr/sbin/maas-provision maas ALL= NOPASSWD: SETENV: /usr/sbin/maas-import-pxe-files diff -Nru maas-1.5+bzr2252/debian/maas-cluster-controller.postinst maas-1.5.4+bzr2294/debian/maas-cluster-controller.postinst --- maas-1.5+bzr2252/debian/maas-cluster-controller.postinst 2014-04-15 16:06:26.000000000 +0000 +++ maas-1.5.4+bzr2294/debian/maas-cluster-controller.postinst 2014-09-19 20:32:28.000000000 +0000 @@ -103,6 +103,7 @@ fi fi echo '0.0.0.0/0:68,69' >/etc/authbind/byuid/$MAAS_UID + echo '::/0,68-69' >>/etc/authbind/byuid/$MAAS_UID chown maas:maas /etc/authbind/byuid/$MAAS_UID chmod 700 /etc/authbind/byuid/$MAAS_UID } diff -Nru maas-1.5+bzr2252/debian/patches/04-enable-armhf-keystone.patch maas-1.5.4+bzr2294/debian/patches/04-enable-armhf-keystone.patch --- maas-1.5+bzr2252/debian/patches/04-enable-armhf-keystone.patch 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/debian/patches/04-enable-armhf-keystone.patch 2014-09-19 20:44:30.000000000 +0000 @@ -0,0 +1,15 @@ +Description: Add hardware enablement for armhf/keystone +Bug: https://bugs.launchpad.net/ubuntu/+source/maas/+bug/1350103 +Upstream: revno: 2634 +--- a/src/provisioningserver/driver/__init__.py ++++ b/src/provisioningserver/driver/__init__.py +@@ -145,6 +145,9 @@ builtin_architectures = [ + Architecture( + name="armhf/generic", description="armhf/generic", + pxealiases=["arm"], kernel_options=["console=ttyAMA0"]), ++ Architecture( ++ name="armhf/keystone", description="armhf/keystone", ++ pxealiases=["arm"]), + # PPC64EL needs a rootdelay for PowerNV. The disk controller + # in the hardware, takes a little bit longer to come up then + # the initrd wants to wait. Set this to 60 seconds, just to diff -Nru maas-1.5+bzr2252/debian/patches/home-directory.patch maas-1.5.4+bzr2294/debian/patches/home-directory.patch --- maas-1.5+bzr2252/debian/patches/home-directory.patch 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/debian/patches/home-directory.patch 2014-12-04 18:59:51.000000000 +0000 @@ -0,0 +1,13 @@ +Description: Fix compatibility with mod-wsgi security update +Bug-Ubuntu: https://bugs.launchpad.net/maas/+bug/1399016 + +Index: maas-1.7.0~beta8+bzr3272/contrib/maas-http.conf +=================================================================== +--- maas-1.7.0~beta8+bzr3272.orig/contrib/maas-http.conf 2014-10-22 12:55:39.000000000 -0400 ++++ maas-1.7.0~beta8+bzr3272/contrib/maas-http.conf 2014-12-04 13:56:05.790551266 -0500 +@@ -1,4 +1,4 @@ +-WSGIDaemonProcess maas user=maas group=maas processes=2 threads=1 display-name=%{GROUP} ++WSGIDaemonProcess maas user=maas group=maas home=/var/lib/maas/ processes=2 threads=1 display-name=%{GROUP} + + # Without this, defining a tag as a malformed xpath expression will hang + # the region controller. diff -Nru maas-1.5+bzr2252/debian/patches/series maas-1.5.4+bzr2294/debian/patches/series --- maas-1.5+bzr2252/debian/patches/series 2014-04-15 16:06:26.000000000 +0000 +++ maas-1.5.4+bzr2294/debian/patches/series 2014-12-04 18:59:51.000000000 +0000 @@ -1,3 +1,5 @@ 01-fix-database-settings.patch 02-pserv-config.patch 03-txlongpoll-config.patch +04-enable-armhf-keystone.patch +home-directory.patch diff -Nru maas-1.5+bzr2252/docs/changelog.rst maas-1.5.4+bzr2294/docs/changelog.rst --- maas-1.5+bzr2252/docs/changelog.rst 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/changelog.rst 2014-09-03 14:18:31.000000000 +0000 @@ -2,6 +2,79 @@ Changelog ========= +1.5.4 +===== + +Bug fix update +-------------- + + - Package fails to install when the default route is through an + aliased/tagged interface (LP: #1350235) + - ERROR Nonce already used (LP: #1190986) + - Add MAAS arm64/xgene support (LP: #1338851) + - Add utopic support (LP: #1337437) + - API documentation for nodegroup op=details missing parameter + (LP: #1331982) + + +1.5.3 +===== + +Bug fix update +-------------- + + - Reduce number of celery tasks emitted when updating a cluster controller + (LP: #1324944) + - Fix VirshSSH template which was referencing invalid attributes + (LP: #1324966) + - Fix a start up problem where a database lock was being taken outside of + a transaction (LP: #1325759) + - Reformat badly formatted Architecture error message (LP: #1301465) + - Final changes to support ppc64el (now known as PowerNV) (LP: #1315154) + + +1.5.2 +===== + +Bug fix update +-------------- + +- Remove workaround for fixed Django bug 1311433 (LP: #1311433) +- Ensure that validation errors are returned when adding a node over + the API and its cluster controller is not contactable. (LP: #1305061) +- Hardware enablement support for PowerKVM +- Shorten the time taken for a cluster to initially connect to the region + via RPC to around 2 seconds (LP: #1317682) +- Faster DHCP leases parser (LP: #1305102) +- Documentation fixed explaining how to enable an ephemeral backdoor + (LP: #1321696) +- Use probe-and-enlist-hardware to enlist all virtual machine inside + a libvirt machine, allow password qemu+ssh connections. + (LP: #1315155, LP: #1315157) +- Rename ppc64el boot loader to PowerKVM (LP: #1315154) + + +1.5.1 +===== + +Bug fix update +-------------- + +For full details see https://launchpad.net/maas/+milestone/1.5.1 + +#1303915 Powering SM15k RESTAPI v2.0 doesn't force PXE boot +#1307780 no armhf commissioning template +#1310076 lost connectivity to a node when using fastpath-installer with precise+hwe-s +#1310082 d-i with precise+hwe-s stops at "Architecture not supported" +#1311151 MAAS imports Trusty's 'rc' images by default. +#1311433 REGRESSION: AttributeError: 'functools.partial' object has no attribute '__module__' +#1313556 API client blocks when deleting a resource +#1314409 parallel juju deployments race on the same maas +#1316396 When stopping a node from the web UI that was started from the API, distro_series is not cleared +#1298784 Vulnerable to user-interface redressing (e.g. clickjacking) +#1308772 maas has no way to specify alternate IP addresses for AMT template +#1300476 Unable to setup BMC/UCS user on Cisco B200 M3 + 1.5 === diff -Nru maas-1.5+bzr2252/docs/cluster-configuration.rst maas-1.5.4+bzr2294/docs/cluster-configuration.rst --- maas-1.5+bzr2252/docs/cluster-configuration.rst 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/cluster-configuration.rst 2014-09-03 14:18:31.000000000 +0000 @@ -48,8 +48,8 @@ below). Any other cluster controllers you set up will show up in the user interface as "pending," until you manually accept them into the MAAS. -To accept a cluster controller, click on the settings "cog" icon at the top -right to visit the settings page: +To accept a cluster controller, visit the "pending clusters" section of the +Clusters page: .. image:: media/cluster-accept.png diff -Nru maas-1.5+bzr2252/docs/conf.py maas-1.5.4+bzr2294/docs/conf.py --- maas-1.5+bzr2252/docs/conf.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/conf.py 2014-09-03 14:18:31.000000000 +0000 @@ -258,10 +258,19 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} +# Gather information about the branch and the build date. +from subprocess import check_output +bzr_last_revision_number = check_output(['bzr', 'revno']) +bzr_last_revision_date = check_output(['bzr', 'version-info', '--template={date}', '--custom']) +bzr_build_date = check_output(['bzr', 'version-info', '--template={build_date}', '--custom']) + # Populate html_context with the variables used in the templates. html_context = { 'add_version_switcher': 'true' if add_version_switcher else 'false', 'versions_json_path': '/'.join(['', doc_prefix, versions_path]), 'doc_prefix': doc_prefix, + 'bzr_last_revision_date': bzr_last_revision_date, + 'bzr_last_revision_number': bzr_last_revision_number, + 'bzr_build_date': bzr_build_date, } diff -Nru maas-1.5+bzr2252/docs/man/maas-import-pxe-files.8.rst maas-1.5.4+bzr2294/docs/man/maas-import-pxe-files.8.rst --- maas-1.5+bzr2252/docs/man/maas-import-pxe-files.8.rst 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/man/maas-import-pxe-files.8.rst 2014-09-03 14:18:31.000000000 +0000 @@ -19,8 +19,8 @@ An easier way to run the script is to trigger it from the MAAS web user interface. To do that, log in to your MAAS as an administrator using a -web browser, click the cogwheel icon in the top right of the page to go -to the Clusters page, and click "Import boot images." This will start +web browser, click the "Clusters" tab at the top of the page to go to +the Clusters page, and click "Import boot images." This will start imports on all cluster controllers simultaneously. The same thing can also be done through the region-controller API, or through the command-line interface. Binary files /tmp/GfGf4d5X19/maas-1.5+bzr2252/docs/media/cluster-accept.png and /tmp/eVXcHw1I30/maas-1.5.4+bzr2294/docs/media/cluster-accept.png differ diff -Nru maas-1.5+bzr2252/docs/_templates/maas/layout.html maas-1.5.4+bzr2294/docs/_templates/maas/layout.html --- maas-1.5+bzr2252/docs/_templates/maas/layout.html 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/_templates/maas/layout.html 2014-09-03 14:18:31.000000000 +0000 @@ -10,11 +10,22 @@
{% endblock %} +{# Remove 'modules' and 'index' from rellinks: they point to + autogenerated code documentation pages that we don't want + to advertise too much. +#} +{%- set rellinks = rellinks[2:] %} + {%- block footer %}

© Copyright {{ copyright }}. Ubuntu and Canonical are registered trademarks of - Canonical Ltd

+ Canonical Ltd. +

+

+ Revision {{ bzr_last_revision_number}} ({{ bzr_last_revision_date }}). + Documentation generation date: {{ bzr_build_date }}. +

{%- endblock %} diff -Nru maas-1.5+bzr2252/docs/_templates/maas/static/css/main.css maas-1.5.4+bzr2294/docs/_templates/maas/static/css/main.css --- maas-1.5+bzr2252/docs/_templates/maas/static/css/main.css 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/_templates/maas/static/css/main.css 2014-09-03 14:18:31.000000000 +0000 @@ -9,7 +9,7 @@ div.document { width: 984px; - margin: 30px auto 0 auto; + margin: 10px auto 0 auto; } div.body h1 { diff -Nru maas-1.5+bzr2252/docs/_templates/maas/static/flasky.css_t maas-1.5.4+bzr2294/docs/_templates/maas/static/flasky.css_t --- maas-1.5+bzr2252/docs/_templates/maas/static/flasky.css_t 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/_templates/maas/static/flasky.css_t 2014-09-03 14:18:31.000000000 +0000 @@ -25,7 +25,7 @@ div.document { width: {{ page_width }}; - margin: 30px auto 0 auto; + margin: 10px auto 0 auto; } div.documentwrapper { @@ -69,6 +69,11 @@ } div.related { + width: {{ page_width }}; + margin: 10px auto 0 auto; +} + +div.related h3 { display: none; } diff -Nru maas-1.5+bzr2252/docs/troubleshooting.rst maas-1.5.4+bzr2294/docs/troubleshooting.rst --- maas-1.5+bzr2252/docs/troubleshooting.rst 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/docs/troubleshooting.rst 2014-09-03 14:18:31.000000000 +0000 @@ -127,13 +127,13 @@ sudo apt-get install --assume-yes bzr bzr branch lp:~maas-maintainers/maas/backdoor-image backdoor-image - imgs=$(echo /var/lib/maas/ephemeral/*/*/*/*/*.img) + imgs=$(echo /var/lib/maas/boot-resources/*/*/*/*/*/root-image) for img in $imgs; do [ -f "$img.dist" ] || cp -a --sparse=always $img $img.dist done for img in $imgs; do - sudo ./backdoor-image -v --user=backdoor --password-auth --password=ubuntu $img + sudo ./backdoor-image/backdoor-image -v --user=backdoor --password-auth --password=ubuntu $img done Inside the ephemeral image diff -Nru maas-1.5+bzr2252/etc/maas/bootresources.yaml maas-1.5.4+bzr2294/etc/maas/bootresources.yaml --- maas-1.5+bzr2252/etc/maas/bootresources.yaml 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/bootresources.yaml 2014-09-03 14:18:31.000000000 +0000 @@ -58,7 +58,7 @@ - release: "trusty" arches: ["i386", "amd64"] subarches: ["generic"] - labels: ["release", "rc"] + labels: ["release"] - release: "precise" arches: ["i386", "amd64"] subarches: ["generic"] diff -Nru maas-1.5+bzr2252/etc/maas/templates/commissioning-user-data/snippets/maas_api_helper.py maas-1.5.4+bzr2294/etc/maas/templates/commissioning-user-data/snippets/maas_api_helper.py --- maas-1.5+bzr2252/etc/maas/templates/commissioning-user-data/snippets/maas_api_helper.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/commissioning-user-data/snippets/maas_api_helper.py 2014-09-03 14:18:31.000000000 +0000 @@ -12,6 +12,7 @@ import sys import time import urllib2 +import uuid import oauth.oauth as oauth import yaml @@ -60,7 +61,7 @@ params = { 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), + 'oauth_nonce': uuid.uuid4().get_hex(), 'oauth_timestamp': timestamp, 'oauth_token': token.key, 'oauth_consumer_key': consumer.key, diff -Nru maas-1.5+bzr2252/etc/maas/templates/commissioning-user-data/snippets/maas_enlist.sh maas-1.5.4+bzr2294/etc/maas/templates/commissioning-user-data/snippets/maas_enlist.sh --- maas-1.5+bzr2252/etc/maas/templates/commissioning-user-data/snippets/maas_enlist.sh 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/commissioning-user-data/snippets/maas_enlist.sh 2014-09-03 14:18:31.000000000 +0000 @@ -61,7 +61,7 @@ get_host_subarchitecture() { local arch=$1 case $arch in - i386|amd64) + i386|amd64|ppc64el) # Skip the call to archdetect as that's what # get_host_architecture does echo generic diff -Nru maas-1.5+bzr2252/etc/maas/templates/dhcp/dhcpd.conf.template maas-1.5.4+bzr2294/etc/maas/templates/dhcp/dhcpd.conf.template --- maas-1.5+bzr2252/etc/maas/templates/dhcp/dhcpd.conf.template 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/dhcp/dhcpd.conf.template 2014-09-03 14:18:31.000000000 +0000 @@ -6,6 +6,7 @@ # the nodegroup's configuration in MAAS to trigger an update. option arch code 93 = unsigned integer 16; # RFC4578 +option path-prefix code 210 = text; #RFC5071 {{for dhcp_subnet in dhcp_subnets}} subnet {{dhcp_subnet['subnet']}} netmask {{dhcp_subnet['subnet_mask']}} { {{bootloader}} diff -Nru maas-1.5+bzr2252/etc/maas/templates/power/amt.template maas-1.5.4+bzr2294/etc/maas/templates/power/amt.template --- maas-1.5+bzr2252/etc/maas/templates/power/amt.template 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/power/amt.template 2014-09-03 14:18:31.000000000 +0000 @@ -2,10 +2,17 @@ # -*- mode: shell-script -*- # # Control a system via amttool +power_address='{{power_address}}' power_change='{{power_change}}' power_pass='{{power_pass}}' ip_address='{{ip_address}}' +# The user specified power_address overrides any automatically determined +# ip_address. +if [ -n "$power_address" ]; then + ip_address=$power_address +fi + echo amt.template starting $* echo ip_address $ip_address diff -Nru maas-1.5+bzr2252/etc/maas/templates/power/mscm.template maas-1.5.4+bzr2294/etc/maas/templates/power/mscm.template --- maas-1.5+bzr2252/etc/maas/templates/power/mscm.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/power/mscm.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,15 @@ +# -*- mode: shell-script -*- +# +# Control a system via Moonshot HP iLO Chassis Manager (MSCM). + +{{py: from provisioningserver.utils import escape_py_literal}} +python - << END +from provisioningserver.drivers.hardware.mscm import power_control_mscm +power_control_mscm( + {{escape_py_literal(power_address) | safe}}, + {{escape_py_literal(power_user) | safe}}, + {{escape_py_literal(power_pass) | safe}}, + {{escape_py_literal(node_id) | safe}}, + {{escape_py_literal(power_change) | safe}}, +) +END diff -Nru maas-1.5+bzr2252/etc/maas/templates/power/ucsm.template maas-1.5.4+bzr2294/etc/maas/templates/power/ucsm.template --- maas-1.5+bzr2252/etc/maas/templates/power/ucsm.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/power/ucsm.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,15 @@ +# -*- mode: shell-script -*- +# +# Control a system via Cisco UCS Manager XML API. + +{{py: from provisioningserver.utils import escape_py_literal}} +python - << END +from provisioningserver.custom_hardware.ucsm import power_control_ucsm +power_control_ucsm( + {{escape_py_literal(power_address) | safe}}, + {{escape_py_literal(power_user) | safe}}, + {{escape_py_literal(power_pass) | safe}}, + {{escape_py_literal(uuid) | safe}}, + {{escape_py_literal(power_change) | safe}}, +) +END diff -Nru maas-1.5+bzr2252/etc/maas/templates/power/virsh.template maas-1.5.4+bzr2294/etc/maas/templates/power/virsh.template --- maas-1.5+bzr2252/etc/maas/templates/power/virsh.template 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/power/virsh.template 2014-09-03 14:18:31.000000000 +0000 @@ -3,50 +3,16 @@ # Control virtual system's "power" through virsh. # -# Parameters. -power_change={{power_change}} -power_address={{power_address}} -power_id={{power_id}} -virsh={{virsh}} - - -# Choose command for virsh to make the requested power change happen. -formulate_power_command() { - if [ ${power_change} = 'on' ] - then - echo 'start' - else - echo 'destroy' - fi -} - - -# Express system's current state as expressed by virsh as "on" or "off". -formulate_power_state() { - case $1 in - 'running') echo 'on' ;; - 'shut off') echo 'off' ;; - *) - echo "Got unknown power state from virsh: '$1'" >&2 - exit 1 - esac -} - - -# Issue command to virsh, for the given system. issue_virsh_command() { - ${virsh} --connect ${power_address} $1 ${power_id} -} - - -# Get the given system's power state: 'on' or 'off'. -get_power_state() { - virsh_state=$(issue_virsh_command domstate) - formulate_power_state ${virsh_state} +python - << END +from provisioningserver.custom_hardware.virsh import power_control_virsh +power_control_virsh( + {{repr(power_address).decode("ascii") | safe}}, + {{repr(power_id).decode("ascii") | safe}}, + {{repr(power_change).decode("ascii") | safe}}, + {{repr(power_pass).decode("ascii") | safe}}, +) +END } - -if [ "$(get_power_state)" != "${power_change}" ] -then - issue_virsh_command $(formulate_power_command) -fi +issue_virsh_command diff -Nru maas-1.5+bzr2252/etc/maas/templates/pxe/config.commissioning.arm64.template maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.commissioning.arm64.template --- maas-1.5+bzr2252/etc/maas/templates/pxe/config.commissioning.arm64.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.commissioning.arm64.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,8 @@ +DEFAULT execute + +LABEL execute + {{# SAY is not implemented in U-Boot }} + KERNEL {{kernel_params | kernel_path }} + INITRD {{kernel_params | initrd_path }} + APPEND {{kernel_params | kernel_command}} + IPAPPEND 2 diff -Nru maas-1.5+bzr2252/etc/maas/templates/pxe/config.commissioning.ppc64el.template maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.commissioning.ppc64el.template --- maas-1.5+bzr2252/etc/maas/templates/pxe/config.commissioning.ppc64el.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.commissioning.ppc64el.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,6 @@ +DEFAULT execute + +LABEL execute + KERNEL {{kernel_params | kernel_path }} + INITRD {{kernel_params | initrd_path }} + APPEND {{kernel_params | kernel_command}} diff -Nru maas-1.5+bzr2252/etc/maas/templates/pxe/config.install.arm64.template maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.install.arm64.template --- maas-1.5+bzr2252/etc/maas/templates/pxe/config.install.arm64.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.install.arm64.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,8 @@ +DEFAULT execute + +LABEL execute + {{# SAY is not implemented in U-Boot }} + KERNEL {{kernel_params | kernel_path }} + INITRD {{kernel_params | initrd_path }} + APPEND {{kernel_params | kernel_command}} + IPAPPEND 2 diff -Nru maas-1.5+bzr2252/etc/maas/templates/pxe/config.install.ppc64el.template maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.install.ppc64el.template --- maas-1.5+bzr2252/etc/maas/templates/pxe/config.install.ppc64el.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.install.ppc64el.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,6 @@ +DEFAULT execute + +LABEL execute + KERNEL {{kernel_params | kernel_path }} + INITRD {{kernel_params | initrd_path }} + APPEND {{kernel_params | kernel_command}} diff -Nru maas-1.5+bzr2252/etc/maas/templates/pxe/config.xinstall.arm64.template maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.xinstall.arm64.template --- maas-1.5+bzr2252/etc/maas/templates/pxe/config.xinstall.arm64.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.xinstall.arm64.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,8 @@ +DEFAULT execute + +LABEL execute + {{# SAY is not implemented in U-Boot }} + KERNEL {{kernel_params | kernel_path }} + INITRD {{kernel_params | initrd_path }} + APPEND {{kernel_params | kernel_command}} + IPAPPEND 2 diff -Nru maas-1.5+bzr2252/etc/maas/templates/pxe/config.xinstall.ppc64el.template maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.xinstall.ppc64el.template --- maas-1.5+bzr2252/etc/maas/templates/pxe/config.xinstall.ppc64el.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/pxe/config.xinstall.ppc64el.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,6 @@ +DEFAULT execute + +LABEL execute + KERNEL {{kernel_params | kernel_path }} + INITRD {{kernel_params | initrd_path }} + APPEND {{kernel_params | kernel_command}} diff -Nru maas-1.5+bzr2252/etc/maas/templates/uefi/config.commissioning.template maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.commissioning.template --- maas-1.5+bzr2252/etc/maas/templates/uefi/config.commissioning.template 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.commissioning.template 2014-09-03 14:18:31.000000000 +0000 @@ -1,10 +1,8 @@ set default="0" set timeout=0 -# Force AMD64 for commissioning as UEFI only supports AMD64 currently. menuentry 'Commission' { echo 'Booting under MAAS direction...' - echo '{{kernel_params() | kernel_command}} BOOTIF=01-'${net_default_mac} - linux {{kernel_params(arch="amd64") | kernel_path }} {{kernel_params(arch="amd64") | kernel_command}} BOOTIF=01-${net_default_mac} - initrd {{kernel_params(arch="amd64") | initrd_path }} + linux {{kernel_params | kernel_path }} {{kernel_params | kernel_command}} BOOTIF=01-${net_default_mac} + initrd {{kernel_params | initrd_path }} } diff -Nru maas-1.5+bzr2252/etc/maas/templates/uefi/config.install.template maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.install.template --- maas-1.5+bzr2252/etc/maas/templates/uefi/config.install.template 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.install.template 2014-09-03 14:18:31.000000000 +0000 @@ -3,7 +3,6 @@ menuentry 'Install' { echo 'Booting under MAAS direction...' - echo '{{kernel_params | kernel_command}} BOOTIF=01-'${net_default_mac} linux {{kernel_params | kernel_path }} {{kernel_params | kernel_command}} BOOTIF=01-${net_default_mac} initrd {{kernel_params | initrd_path }} } diff -Nru maas-1.5+bzr2252/etc/maas/templates/uefi/config.local.amd64.template maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.local.amd64.template --- maas-1.5+bzr2252/etc/maas/templates/uefi/config.local.amd64.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.local.amd64.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,8 @@ +set default="0" +set timeout=0 + +menuentry 'Local' { + echo 'Booting local disk...' + search --set=root --file /efi/ubuntu/grub.cfg + configfile /efi/ubuntu/grub.cfg +} diff -Nru maas-1.5+bzr2252/etc/maas/templates/uefi/config.local.ppc64el.template maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.local.ppc64el.template --- maas-1.5+bzr2252/etc/maas/templates/uefi/config.local.ppc64el.template 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.local.ppc64el.template 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,8 @@ +set default="0" +set timeout=0 + +menuentry 'Local' { + echo 'Booting local disk...' + search --set=root --file /boot/grub/grub.cfg + configfile /boot/grub/grub.cfg +} diff -Nru maas-1.5+bzr2252/etc/maas/templates/uefi/config.local.template maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.local.template --- maas-1.5+bzr2252/etc/maas/templates/uefi/config.local.template 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.local.template 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -set default="0" -set timeout=0 - -menuentry 'Local' { - echo 'Booting local disk ...' - search --set=root --file /efi/ubuntu/grub.cfg - configfile /efi/ubuntu/grub.cfg -} diff -Nru maas-1.5+bzr2252/etc/maas/templates/uefi/config.xinstall.template maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.xinstall.template --- maas-1.5+bzr2252/etc/maas/templates/uefi/config.xinstall.template 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/etc/maas/templates/uefi/config.xinstall.template 2014-09-03 14:18:31.000000000 +0000 @@ -3,7 +3,6 @@ menuentry 'Install' { echo 'Booting under MAAS direction...' - echo '{{kernel_params | kernel_command}} BOOTIF=01-'${net_default_mac} linux {{kernel_params | kernel_path }} {{kernel_params | kernel_command}} BOOTIF=01-${net_default_mac} initrd {{kernel_params | initrd_path }} } diff -Nru maas-1.5+bzr2252/Makefile maas-1.5.4+bzr2294/Makefile --- maas-1.5+bzr2252/Makefile 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/Makefile 2014-09-03 14:18:31.000000000 +0000 @@ -164,7 +164,7 @@ doc: bin/sphinx docs/api.rst bin/sphinx -doc-with-versions: bin/sphinx +doc-with-versions: bin/sphinx docs/api.rst cd docs/_build; make SPHINXOPTS="-A add_version_switcher=true" html man: $(patsubst docs/man/%.rst,man/%,$(wildcard docs/man/*.rst)) @@ -344,8 +344,8 @@ # has a bug and always considers apt-source tarballs before the specified # branch. So instead, export to a local tarball which is always found. # Make sure debhelper and dh-apport packages are installed before using this. -PACKAGING := $(CURDIR)/../packaging.trunk -PACKAGING_BRANCH := lp:~maas-maintainers/maas/packaging +PACKAGING := $(CURDIR)/../packaging.trusty +PACKAGING_BRANCH := lp:~maas-maintainers/maas/packaging.trusty package_branch: @echo Downloading/refreshing packaging branch... @@ -363,11 +363,11 @@ @bzr export --root=maas-$(VER).orig ../build-area/$(TARBALL) $(CURDIR) package: package_export - bzr bd --merge $(PACKAGING) -- -uc -us + bzr bd --merge $(PACKAGING) --result-dir=../build-area -- -uc -us @echo Binary packages built, see parent directory. source_package: package_export - bzr bd --merge $(PACKAGING) -- -S -uc -us + bzr bd --merge $(PACKAGING) --result-dir=../build-area -- -S -uc -us @echo Source package built, see parent directory. # diff -Nru maas-1.5+bzr2252/required-packages/base maas-1.5.4+bzr2294/required-packages/base --- maas-1.5+bzr2252/required-packages/base 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/required-packages/base 2014-09-03 14:18:31.000000000 +0000 @@ -35,6 +35,7 @@ python-oops-datedir-repo python-oops-twisted python-oops-wsgi +python-pexpect python-psycopg2 python-pyinotify python-seamicroclient diff -Nru maas-1.5+bzr2252/src/apiclient/maas_client.py maas-1.5.4+bzr2294/src/apiclient/maas_client.py --- maas-1.5+bzr2252/src/apiclient/maas_client.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/apiclient/maas_client.py 2014-09-03 14:18:31.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2012 Canonical Ltd. This software is licensed under the +# Copyright 2012-2014 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """MAAS OAuth API connection library.""" @@ -21,6 +21,7 @@ import gzip from io import BytesIO import urllib2 +import uuid from apiclient.encode_json import encode_json_data from apiclient.multipart import encode_multipart_data @@ -45,7 +46,8 @@ with the signature. """ oauth_request = oauth.OAuthRequest.from_consumer_and_token( - self.consumer_token, token=self.resource_token, http_url=url) + self.consumer_token, token=self.resource_token, http_url=url, + parameters={'oauth_nonce': uuid.uuid4().get_hex()}) oauth_request.sign_request( oauth.OAuthSignatureMethod_PLAINTEXT(), self.consumer_token, self.resource_token) @@ -240,5 +242,7 @@ def delete(self, path): """Dispatch a DELETE on the resource at `path`.""" url, headers, body = self._formulate_change(path, {}) + # The body will be empty, but it must be passed. Otherwise, the + # request will hang while trying to read a response (bug 1313556). return self.dispatcher.dispatch_query( - url, method="DELETE", headers=headers) + url, method="DELETE", headers=headers, data=body) diff -Nru maas-1.5+bzr2252/src/apiclient/tests/test_maas_client.py maas-1.5.4+bzr2294/src/apiclient/tests/test_maas_client.py --- maas-1.5+bzr2252/src/apiclient/tests/test_maas_client.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/apiclient/tests/test_maas_client.py 2014-09-03 14:18:31.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2012, 2013 Canonical Ltd. This software is licensed under the +# Copyright 2012-2014 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Test MAAS HTTP API client.""" @@ -346,3 +346,10 @@ self.assertEqual(client._make_url(path), request['request_url']) self.assertIn('Authorization', request['headers']) self.assertEqual('DELETE', request['method']) + + def test_delete_passes_body(self): + # A DELETE request should have an empty body. But we can't just leave + # the body out altogether, or the request will hang (bug 1313556). + client = make_client() + client.delete(make_path()) + self.assertIsNotNone(client.dispatcher.last_call['data']) diff -Nru maas-1.5+bzr2252/src/maas/development.py maas-1.5.4+bzr2294/src/maas/development.py --- maas-1.5+bzr2252/src/maas/development.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maas/development.py 2014-09-03 14:18:31.000000000 +0000 @@ -24,6 +24,7 @@ ) from metadataserver.address import guess_server_address import provisioningserver.config +from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE # We expect the following settings to be overridden. They are mentioned here # to silence lint warnings. @@ -68,7 +69,10 @@ # For PostgreSQL, a "hostname" starting with a slash indicates a # Unix socket directory. 'HOST': abspath('db'), - } + 'OPTIONS': { + 'isolation_level': ISOLATION_LEVEL_SERIALIZABLE, + }, + }, } # Absolute filesystem path to the directory that will hold user-uploaded files. diff -Nru maas-1.5+bzr2252/src/maas/settings.py maas-1.5.4+bzr2294/src/maas/settings.py --- maas-1.5+bzr2252/src/maas/settings.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maas/settings.py 2014-09-03 14:18:31.000000000 +0000 @@ -21,6 +21,7 @@ import django.template from maas import import_local_settings from metadataserver.address import guess_server_address +from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE django.template.add_to_builtins('django.templatetags.future') @@ -138,7 +139,10 @@ # Unix socket directory. 'HOST': '', 'PORT': '', - } + 'OPTIONS': { + 'isolation_level': ISOLATION_LEVEL_SERIALIZABLE, + }, + }, } # Local time zone for this installation. Choices can be found here: @@ -249,6 +253,7 @@ 'maasserver.middleware.ExternalComponentsMiddleware', 'metadataserver.middleware.MetadataErrorsMiddleware', 'django.middleware.transaction.TransactionMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'maasserver.middleware.ExceptionLoggerMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', diff -Nru maas-1.5+bzr2252/src/maasserver/api.py maas-1.5.4+bzr2294/src/maasserver/api.py --- maas-1.5+bzr2252/src/maasserver/api.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/api.py 2014-09-03 14:18:31.000000000 +0000 @@ -427,7 +427,6 @@ """Release a node. Opposite of `NodesHandler.acquire`.""" node = Node.objects.get_node_or_404( system_id=system_id, user=request.user, perm=NODE_PERMISSION.EDIT) - node.set_distro_series(series='') if node.status == NODE_STATUS.READY: # Nothing to do. This may be a redundant retry, and the # postcondition is achieved, so call this success. @@ -1581,6 +1580,8 @@ Returns a ``{system_id: {detail_type: xml, ...}, ...}`` map, where ``detail_type`` is something like "lldp" or "lshw". + :param system_ids: System ids of nodes for which to get system details. + Note that this is returned as BSON and not JSON. This is for efficiency, but mainly because JSON can't do binary content without applying additional encoding like base-64. @@ -1685,8 +1686,8 @@ def probe_and_enlist_hardware(self, request, uuid): """Add special hardware types. - :param model: The type of special hardware, currently only - 'seamicro15k' is supported. + :param model: The type of special hardware, 'seamicro15k' and + 'virsh' is supported. :type model: unicode The following are only required if you are probing a seamicro15k: @@ -1703,6 +1704,17 @@ :param power_control: The power_control to use, either ipmi (default) or restapi. :type power_control: unicode + + The following are only required if you are probing a virsh: + + :param power_address: The connection string to virsh. + :type power_address: unicode + + The following are optional if you are probing a virsh: + + :param power_pass: The password to use, when qemu+ssh is given as a + connection string and ssh key authentication is not being used. + :type power_pass: unicode """ nodegroup = get_object_or_404(NodeGroup, uuid=uuid) @@ -1717,11 +1729,64 @@ nodegroup.add_seamicro15k( mac, username, password, power_control=power_control) + elif model == 'powerkvm' or model == 'virsh': + poweraddr = get_mandatory_param(request.data, 'power_address') + password = get_optional_param( + request.data, 'power_pass', default=None) + + nodegroup.add_virsh(poweraddr, password=password) else: return HttpResponse(status=httplib.BAD_REQUEST) return HttpResponse(status=httplib.OK) + @admin_method + @operation(idempotent=False) + def probe_and_enlist_ucsm(self, request, uuid): + """Add the nodes from a Cisco UCS Manager. + + :param : The URL of the UCS Manager API. + :type url: unicode + :param username: The username for the API. + :type username: unicode + :param password: The password for the API. + :type password: unicode + + """ + nodegroup = get_object_or_404(NodeGroup, uuid=uuid) + + url = get_mandatory_param(request.data, 'url') + username = get_mandatory_param(request.data, 'username') + password = get_mandatory_param(request.data, 'password') + + nodegroup.enlist_nodes_from_ucsm(url, username, password) + + return HttpResponse(status=httplib.OK) + + @admin_method + @operation(idempotent=False) + def probe_and_enlist_mscm(self, request, uuid): + """Add the nodes from a Moonshot HP iLO Chassis Manager (MSCM). + + :param host: IP Address for the MSCM. + :type host: unicode + :param username: The username for the MSCM. + :type username: unicode + :param password: The password for the MSCM. + :type password: unicode + + """ + nodegroup = get_object_or_404(NodeGroup, uuid=uuid) + + host = get_mandatory_param(request.data, 'host') + username = get_mandatory_param(request.data, 'username') + password = get_mandatory_param(request.data, 'password') + + nodegroup.enlist_nodes_from_mscm(host, username, password) + + return HttpResponse(status=httplib.OK) + + DISPLAYED_NODEGROUPINTERFACE_FIELDS = ( 'ip', 'management', 'interface', 'subnet_mask', 'broadcast_ip', 'ip_range_low', 'ip_range_high') diff -Nru maas-1.5+bzr2252/src/maasserver/dhcp.py maas-1.5.4+bzr2294/src/maasserver/dhcp.py --- maas-1.5+bzr2252/src/maasserver/dhcp.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/dhcp.py 2014-09-03 14:18:31.000000000 +0000 @@ -22,6 +22,7 @@ from netaddr import IPAddress from provisioningserver.tasks import ( restart_dhcp_server, + stop_dhcp_server, write_dhcp_config, ) @@ -52,7 +53,15 @@ from maasserver.dns import get_dns_server_address interfaces = get_interfaces_managed_by(nodegroup) - if interfaces is None: + if interfaces in [None, []]: + # interfaces being None means the cluster isn't accepted: stop + # the DHCP server in case it case started. + # interfaces being [] means there is no interface configured: stop + # the DHCP server; Note that a config generated with this setup + # would not be valid and would result in the DHCP + # server failing with the error: "Not configured to listen on any + # interfaces!." + stop_dhcp_server.apply_async(queue=nodegroup.work_queue) return # Make sure this nodegroup has a key to communicate with the dhcp diff -Nru maas-1.5+bzr2252/src/maasserver/enum.py maas-1.5.4+bzr2294/src/maasserver/enum.py --- maas-1.5+bzr2252/src/maasserver/enum.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/enum.py 2014-09-03 14:18:31.000000000 +0000 @@ -93,21 +93,19 @@ #: precise = 'precise' #: - quantal = 'quantal' - #: - raring = 'raring' - #: saucy = 'saucy' #: trusty = 'trusty' + #: + utopic = 'utopic' + DISTRO_SERIES_CHOICES = ( (DISTRO_SERIES.default, 'Default Ubuntu Release'), (DISTRO_SERIES.precise, 'Ubuntu 12.04 LTS "Precise Pangolin"'), - (DISTRO_SERIES.quantal, 'Ubuntu 12.10 "Quantal Quetzal"'), - (DISTRO_SERIES.raring, 'Ubuntu 13.04 "Raring Ringtail"'), (DISTRO_SERIES.saucy, 'Ubuntu 13.10 "Saucy Salamander"'), (DISTRO_SERIES.trusty, 'Ubuntu 14.04 LTS "Trusty Tahr"'), + (DISTRO_SERIES.utopic, 'Ubuntu 14.10 "Utopic Unicorn"'), ) diff -Nru maas-1.5+bzr2252/src/maasserver/forms.py maas-1.5.4+bzr2294/src/maasserver/forms.py --- maas-1.5+bzr2252/src/maasserver/forms.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/forms.py 2014-09-03 14:18:31.000000000 +0000 @@ -118,6 +118,17 @@ BLANK_CHOICE = ('', '-------') +def set_form_error(form, field_name, error_value): + """Set an error on a form's field. + + This utility method encapsulates Django's arguably awkward way + of settings errors inside a form's clean()/is_valid() method. This + method will override any previously-registered error for 'field_name'. + """ + # Hey Django devs, this is a crap API to set errors. + form.errors[field_name] = form.error_class([error_value]) + + def remove_None_values(data): """Return a new dictionary without the keys corresponding to None values. """ @@ -249,8 +260,8 @@ def is_valid(self): is_valid = super(NodeForm, self).is_valid() if len(list_all_usable_architectures()) == 0: - self.errors['architecture'] = ( - [NO_ARCHITECTURES_AVAILABLE]) + set_form_error( + self, "architecture", NO_ARCHITECTURES_AVAILABLE) is_valid = False return is_valid @@ -420,12 +431,11 @@ try: get_power_types([self._get_nodegroup()]) except ClusterUnavailable as e: - # Hey Django devs, this is a crap API to set errors. - self._errors["power_type"] = self.error_class( - [CLUSTER_NOT_AVAILABLE + e.args[0]]) + set_form_error( + self, "power_type", CLUSTER_NOT_AVAILABLE + e.args[0]) # If power_type is not set and power_parameters_skip_check is not # on, reset power_parameters (set it to the empty string). - no_power_type = cleaned_data['power_type'] == '' + no_power_type = cleaned_data.get('power_type', '') == '' if no_power_type and not skip_check: cleaned_data['power_parameters'] = '' return cleaned_data diff -Nru maas-1.5+bzr2252/src/maasserver/models/nodegroup.py maas-1.5.4+bzr2294/src/maasserver/models/nodegroup.py --- maas-1.5+bzr2252/src/maasserver/models/nodegroup.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/models/nodegroup.py 2014-09-03 14:18:31.000000000 +0000 @@ -41,6 +41,9 @@ from provisioningserver.tasks import ( add_new_dhcp_host_map, add_seamicro15k, + add_virsh, + enlist_nodes_from_mscm, + enlist_nodes_from_ucsm, import_boot_images, report_boot_images, ) @@ -286,6 +289,35 @@ args = (mac, username, password, power_control) add_seamicro15k.apply_async(queue=self.uuid, args=args) + def add_virsh(self, poweraddr, password=None): + """ Add all of the virtual machines inside a virsh controller. + + :param poweraddr: virsh connection string + :param password: ssh password + """ + args = (poweraddr, password) + add_virsh.apply_async(queue=self.uuid, args=args) + + def enlist_nodes_from_ucsm(self, url, username, password): + """ Add the servers from a Cicso UCS Manager. + + :param URL: URL of the Cisco UCS Manager HTTP-XML API. + :param username: username for UCS Manager. + :param password: password for UCS Manager. + """ + args = (url, username, password) + enlist_nodes_from_ucsm.apply_async(queue=self.uuid, args=args) + + def enlist_nodes_from_mscm(self, host, username, password): + """ Add the servers from a Moonshot HP iLO Chassis Manager. + + :param host: IP address for the MSCM. + :param username: username for MSCM. + :param password: password for MSCM. + """ + args = (host, username, password) + enlist_nodes_from_mscm.apply_async(queue=self.uuid, args=args) + def add_dhcp_host_maps(self, new_leases): if len(new_leases) > 0 and len(self.get_managed_interfaces()) > 0: # XXX JeroenVermeulen 2012-08-21, bug=1039362: the DHCP diff -Nru maas-1.5+bzr2252/src/maasserver/models/node.py maas-1.5.4+bzr2294/src/maasserver/models/node.py --- maas-1.5+bzr2252/src/maasserver/models/node.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/models/node.py 2014-09-03 14:18:31.000000000 +0000 @@ -758,7 +758,6 @@ power_params = {} power_params.setdefault('system_id', self.system_id) - power_params.setdefault('virsh', '/usr/bin/virsh') power_params.setdefault('fence_cdu', '/usr/sbin/fence_cdu') power_params.setdefault('ipmipower', '/usr/sbin/ipmipower') power_params.setdefault('ipmitool', '/usr/bin/ipmitool') @@ -769,6 +768,7 @@ power_params.setdefault('username', '') power_params.setdefault('power_id', self.system_id) power_params.setdefault('power_driver', '') + power_params.setdefault('power_pass', '') # The "mac" parameter defaults to the node's primary MAC # address, but only if not already set. @@ -797,6 +797,7 @@ self.token = None self.agent_name = '' self.set_netboot() + self.distro_series = '' self.save() def set_netboot(self, on=True): @@ -859,3 +860,8 @@ "expression. This expression must instead be updated to set " "this node to install with the fast installer.") self.tags.add(uti_tag) + + def split_arch(self): + """Return architecture and subarchitecture, as a tuple.""" + arch, subarch = self.architecture.split('/') + return (arch, subarch) diff -Nru maas-1.5+bzr2252/src/maasserver/models/tests/test_node.py maas-1.5.4+bzr2294/src/maasserver/models/tests/test_node.py --- maas-1.5+bzr2252/src/maasserver/models/tests/test_node.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/models/tests/test_node.py 2014-09-03 14:18:31.000000000 +0000 @@ -162,7 +162,7 @@ def test_set_get_distro_series_returns_series(self): node = factory.make_node() - series = DISTRO_SERIES.quantal + series = DISTRO_SERIES.utopic node.set_distro_series(series) self.assertEqual(series, node.get_distro_series()) @@ -471,6 +471,13 @@ node.release() self.assertTrue(node.netboot) + def test_release_clears_distro_series(self): + node = factory.make_node( + status=NODE_STATUS.ALLOCATED, owner=factory.make_user()) + node.set_distro_series(series=DISTRO_SERIES.utopic) + node.release() + self.assertEqual("", node.distro_series) + def test_release_powers_off_node(self): # Test that releasing a node causes a 'power_off' celery job. node = factory.make_node( @@ -755,6 +762,13 @@ "The use-fastpath-installer tag is defined with an expression", unicode(error)) + def test_split_arch_returns_arch_as_tuple(self): + main_arch = factory.make_name('arch') + sub_arch = factory.make_name('subarch') + full_arch = '%s/%s' % (main_arch, sub_arch) + node = factory.make_node(architecture=full_arch) + self.assertEqual((main_arch, sub_arch), node.split_arch()) + class NodeRoutersTest(MAASServerTestCase): diff -Nru maas-1.5+bzr2252/src/maasserver/rpc/regionservice.py maas-1.5.4+bzr2294/src/maasserver/rpc/regionservice.py --- maas-1.5+bzr2252/src/maasserver/rpc/regionservice.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/rpc/regionservice.py 2014-09-03 14:18:31.000000000 +0000 @@ -325,8 +325,8 @@ @synchronous @synchronised(lock) - @synchronised(locks.eventloop) @transactional + @synchronised(locks.eventloop) def prepare(self): """Ensure that the ``eventloops`` table exists. diff -Nru maas-1.5+bzr2252/src/maasserver/rpc/tests/test_regionservice.py maas-1.5.4+bzr2294/src/maasserver/rpc/tests/test_regionservice.py --- maas-1.5+bzr2252/src/maasserver/rpc/tests/test_regionservice.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/rpc/tests/test_regionservice.py 2014-09-03 14:18:31.000000000 +0000 @@ -49,6 +49,7 @@ common, exceptions, ) +from provisioningserver.rpc.interfaces import IConnection from provisioningserver.rpc.region import ( Identify, ReportBootImages, @@ -82,7 +83,7 @@ from twisted.internet.threads import deferToThread from twisted.protocols import amp from twisted.python import log -from unittest import skip +from zope.interface.verify import verifyObject class TestRegionProtocol_Identify(MAASTestCase): @@ -196,10 +197,6 @@ return d.addCallback(check) -from provisioningserver.rpc.interfaces import IConnection -from zope.interface.verify import verifyObject - - class TestRegionServer(MAASServerTestCase): def test_interfaces(self): @@ -642,7 +639,6 @@ cursor.execute("SELECT * FROM eventloops") self.assertEqual([], list(cursor)) - @skip("XXX gmb 2014-04-15 bug=1308069: Fails spuriously.") def test_prepare_holds_startup_lock(self): # Creating tables in PostgreSQL is a transactional operation # like any other. If the isolation level is not sufficient - the diff -Nru maas-1.5+bzr2252/src/maasserver/start_up.py maas-1.5.4+bzr2294/src/maasserver/start_up.py --- maas-1.5+bzr2252/src/maasserver/start_up.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/start_up.py 2014-09-03 14:18:31.000000000 +0000 @@ -18,7 +18,10 @@ from textwrap import dedent -from django.db import connection +from django.db import ( + connection, + transaction, + ) from maasserver import ( eventloop, locks, @@ -51,8 +54,9 @@ but this method uses file-based locking to ensure that the methods it calls internally are not ran concurrently. """ - with locks.startup: - inner_start_up() + with transaction.atomic(): + with locks.startup: + inner_start_up() eventloop.start().wait(10) diff -Nru maas-1.5+bzr2252/src/maasserver/testing/tests/test_rabbit.py maas-1.5.4+bzr2294/src/maasserver/testing/tests/test_rabbit.py --- maas-1.5+bzr2252/src/maasserver/testing/tests/test_rabbit.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/testing/tests/test_rabbit.py 2014-09-03 14:18:31.000000000 +0000 @@ -26,7 +26,8 @@ def test_patch(self): config = RabbitServerResources( hostname=factory.getRandomString(), - port=factory.getRandomPort()) + port=factory.getRandomPort(), + dist_port=factory.getRandomPort()) self.useFixture(config) self.useFixture(RabbitServerSettings(config)) self.assertEqual( diff -Nru maas-1.5+bzr2252/src/maasserver/tests/test_api_nodegroup.py maas-1.5.4+bzr2294/src/maasserver/tests/test_api_nodegroup.py --- maas-1.5+bzr2252/src/maasserver/tests/test_api_nodegroup.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/tests/test_api_nodegroup.py 2014-09-03 14:18:31.000000000 +0000 @@ -52,6 +52,7 @@ from maasserver.testing.testcase import MAASServerTestCase from maastesting.celery import CeleryFixture from maastesting.fakemethod import FakeMethod +from maastesting.matchers import MockCalledOnceWith from metadataserver.fields import Bin from metadataserver.models import ( commissioningscript, @@ -433,6 +434,58 @@ httplib.BAD_REQUEST, response.status_code, explain_unexpected_response(httplib.BAD_REQUEST, response)) + def test_probe_and_enlist_ucsm_adds_ucsm(self): + nodegroup = factory.make_node_group() + url = 'http://url' + username = factory.make_name('user') + password = factory.make_name('password') + self.become_admin() + + mock = self.patch(nodegroup_module, 'enlist_nodes_from_ucsm') + + response = self.client.post( + reverse('nodegroup_handler', args=[nodegroup.uuid]), + { + 'op': 'probe_and_enlist_ucsm', + 'url': url, + 'username': username, + 'password': password, + }) + + self.assertEqual( + httplib.OK, response.status_code, + explain_unexpected_response(httplib.OK, response)) + + args = (url, username, password) + matcher = MockCalledOnceWith(queue=nodegroup.uuid, args=args) + self.assertThat(mock.apply_async, matcher) + + def test_probe_and_enlist_mscm_adds_mscm(self): + nodegroup = factory.make_node_group() + host = 'http://host' + username = factory.make_name('user') + password = factory.make_name('password') + self.become_admin() + + mock = self.patch(nodegroup_module, 'enlist_nodes_from_mscm') + + response = self.client.post( + reverse('nodegroup_handler', args=[nodegroup.uuid]), + { + 'op': 'probe_and_enlist_mscm', + 'host': host, + 'username': username, + 'password': password, + }) + + self.assertEqual( + httplib.OK, response.status_code, + explain_unexpected_response(httplib.OK, response)) + + args = (host, username, password) + matcher = MockCalledOnceWith(queue=nodegroup.uuid, args=args) + self.assertThat(mock.apply_async, matcher) + class TestNodeGroupAPIAuth(MAASServerTestCase): """Authorization tests for nodegroup API.""" diff -Nru maas-1.5+bzr2252/src/maasserver/tests/test_api_nodes.py maas-1.5.4+bzr2294/src/maasserver/tests/test_api_nodes.py --- maas-1.5+bzr2252/src/maasserver/tests/test_api_nodes.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/tests/test_api_nodes.py 2014-09-03 14:18:31.000000000 +0000 @@ -19,12 +19,14 @@ import random from django.core.urlresolvers import reverse +from maasserver import forms from maasserver.enum import ( NODE_STATUS, NODE_STATUS_CHOICES_DICT, NODEGROUP_STATUS, NODEGROUPINTERFACE_MANAGEMENT, ) +from maasserver.exceptions import ClusterUnavailable from maasserver.fields import MAC from maasserver.models import Node from maasserver.models.user import ( @@ -175,6 +177,32 @@ NODE_STATUS.DECLARED, Node.objects.get(system_id=system_id).status) + def test_POST_new_when_no_RPC_to_cluster_defaults_empty_power(self): + # Test for bug 1305061, if there is no cluster RPC connection + # then make sure that power_type is defaulted to the empty + # string rather than being entirely absent, which results in a + # crash. + cluster_error = factory.make_name("cluster error") + self.patch(forms, 'get_power_types').side_effect = ( + ClusterUnavailable(cluster_error)) + self.become_admin() + # The patching behind the scenes to avoid *real* RPC is + # complex and the available power types is actually a + # valid set, so use an invalid type to trigger the bug here. + power_type = factory.make_name("power_type") + response = self.client.post( + reverse('nodes_handler'), + { + 'op': 'new', + 'autodetect_nodegroup': '1', + 'architecture': make_usable_architecture(self), + 'mac_addresses': ['aa:bb:cc:dd:ee:ff'], + 'power_type': power_type, + }) + self.assertEqual(httplib.BAD_REQUEST, response.status_code) + validation_errors = json.loads(response.content)['power_type'] + self.assertIn(cluster_error, validation_errors[0]) + def test_GET_list_lists_nodes(self): # The api allows for fetching the list of Nodes. node1 = factory.make_node() diff -Nru maas-1.5+bzr2252/src/maasserver/tests/test_dhcp.py maas-1.5.4+bzr2294/src/maasserver/tests/test_dhcp.py --- maas-1.5+bzr2252/src/maasserver/tests/test_dhcp.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/tests/test_dhcp.py 2014-09-03 14:18:31.000000000 +0000 @@ -33,7 +33,6 @@ from maasserver.testing.testcase import MAASServerTestCase from maasserver.utils import map_enum from maastesting.celery import CeleryFixture -from mock import ANY from netaddr import ( IPAddress, IPNetwork, @@ -72,6 +71,16 @@ {status: None for status in unaccepted_statuses}, managed_interfaces) + def test_configure_dhcp_stops_server_if_no_managed_interface(self): + self.patch(settings, "DHCP_CONNECT", True) + self.patch(dhcp, 'stop_dhcp_server') + nodegroup = factory.make_node_group( + status=NODEGROUP_STATUS.ACCEPTED, + management=NODEGROUPINTERFACE_MANAGEMENT.UNMANAGED, + ) + configure_dhcp(nodegroup) + self.assertEqual(1, dhcp.stop_dhcp_server.apply_async.call_count) + def test_configure_dhcp_obeys_DHCP_CONNECT(self): self.patch(settings, "DHCP_CONNECT", False) self.patch(dhcp, 'write_dhcp_config') @@ -205,19 +214,6 @@ args, kwargs = task.subtask.call_args self.assertEqual(nodegroup.work_queue, kwargs['options']['queue']) - def test_write_dhcp_config_called_when_no_managed_interfaces(self): - nodegroup = factory.make_node_group( - status=NODEGROUP_STATUS.ACCEPTED, - management=NODEGROUPINTERFACE_MANAGEMENT.DHCP) - [interface] = nodegroup.nodegroupinterface_set.all() - self.patch(settings, "DHCP_CONNECT", True) - self.patch(tasks, 'sudo_write_file') - self.patch(dhcp, 'write_dhcp_config') - interface.management = NODEGROUPINTERFACE_MANAGEMENT.UNMANAGED - interface.save() - dhcp.write_dhcp_config.apply_async.assert_called_once_with( - queue=nodegroup.work_queue, kwargs=ANY) - def test_dhcp_config_gets_written_when_interface_IP_changes(self): nodegroup = factory.make_node_group(status=NODEGROUP_STATUS.ACCEPTED) [interface] = nodegroup.nodegroupinterface_set.all() @@ -318,6 +314,9 @@ factory.make_node_group(status=NODEGROUP_STATUS.ACCEPTED) for x in range(num_inactive_nodegroups): factory.make_node_group(status=NODEGROUP_STATUS.PENDING) + # Silence stop_dhcp_server: it will be called for the inactive + # nodegroups. + self.patch(dhcp, 'stop_dhcp_server') self.patch(settings, "DHCP_CONNECT", True) self.patch(dhcp, 'write_dhcp_config') diff -Nru maas-1.5+bzr2252/src/maasserver/tests/test_preseed.py maas-1.5.4+bzr2294/src/maasserver/tests/test_preseed.py --- maas-1.5+bzr2252/src/maasserver/tests/test_preseed.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/tests/test_preseed.py 2014-09-03 14:18:31.000000000 +0000 @@ -62,6 +62,7 @@ AllMatch, Contains, ContainsAll, + HasLength, IsInstance, MatchesAll, Not, @@ -614,6 +615,60 @@ ] )) + def make_fastpath_node(self, main_arch=None): + """Return a `Node`, with FPI enabled, and the given main architecture. + + :param main_arch: A main architecture, such as `i386` or `armhf`. A + subarchitecture will be made up. + """ + if main_arch is None: + main_arch = factory.make_name('arch') + arch = '%s/%s' % (main_arch, factory.make_name('subarch')) + node = factory.make_node(architecture=arch) + node.use_fastpath_installer() + return node + + def extract_archive_setting(self, userdata): + """Extract the `ubuntu_archive` setting from `userdata`.""" + userdata_lines = [] + for line in userdata.splitlines(): + line = line.strip() + if line.startswith('ubuntu_archive'): + userdata_lines.append(line) + self.assertThat(userdata_lines, HasLength(1)) + [userdata_line] = userdata_lines + key, value = userdata_line.split(':', 1) + return value.strip() + + def summarise_url(self, url): + """Return just the hostname and path from `url`, normalised.""" + # This is needed because the userdata deliberately makes some minor + # changes to the archive URLs, making it harder to recognise which + # archive they use: slashes are added, schemes are hard-coded. + parsed_result = urlparse(url) + return parsed_result.netloc, parsed_result.path.strip('/') + + def test_get_curtin_config_uses_main_archive_for_i386(self): + node = self.make_fastpath_node('i386') + userdata = get_curtin_config(node) + self.assertEqual( + self.summarise_url(Config.objects.get_config('main_archive')), + self.summarise_url(self.extract_archive_setting(userdata))) + + def test_get_curtin_config_uses_main_archive_for_amd64(self): + node = self.make_fastpath_node('amd64') + userdata = get_curtin_config(node) + self.assertEqual( + self.summarise_url(Config.objects.get_config('main_archive')), + self.summarise_url(self.extract_archive_setting(userdata))) + + def test_get_curtin_config_uses_ports_archive_for_other_arch(self): + node = self.make_fastpath_node() + userdata = get_curtin_config(node) + self.assertEqual( + self.summarise_url(Config.objects.get_config('ports_archive')), + self.summarise_url(self.extract_archive_setting(userdata))) + def test_get_curtin_context(self): node = factory.make_node() node.use_fastpath_installer() diff -Nru maas-1.5+bzr2252/src/maasserver/utils/dblocks.py maas-1.5.4+bzr2294/src/maasserver/utils/dblocks.py --- maas-1.5+bzr2252/src/maasserver/utils/dblocks.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/utils/dblocks.py 2014-09-03 14:18:31.000000000 +0000 @@ -14,6 +14,8 @@ __metaclass__ = type __all__ = [ "DatabaseLock", + "DatabaseLockAttemptOutsideTransaction", + "DatabaseLockNotHeld", ] from contextlib import closing @@ -26,6 +28,19 @@ classid = 20120116 +class DatabaseLockAttemptOutsideTransaction(Exception): + """A locking attempt was made outside of a transaction. + + :class:`DatabaseLock` should only be used within a transaction. + Django agressively closes connections outside of atomic blocks to + the extent that session-level locks are rendered unreliable at best. + """ + + +class DatabaseLockNotHeld(Exception): + """A particular lock was not held.""" + + class DatabaseLock(tuple): """An advisory lock held in the database. @@ -58,12 +73,16 @@ return super(cls, DatabaseLock).__new__(cls, (classid, objid)) def __enter__(self): + if not connection.in_atomic_block: + raise DatabaseLockAttemptOutsideTransaction(self) with closing(connection.cursor()) as cursor: cursor.execute("SELECT pg_advisory_lock(%s, %s)", self) def __exit__(self, *exc_info): with closing(connection.cursor()) as cursor: cursor.execute("SELECT pg_advisory_unlock(%s, %s)", self) + if cursor.fetchone() != (True,): + raise DatabaseLockNotHeld(self) def __repr__(self): return b"<%s classid=%d objid=%d>" % ( @@ -71,8 +90,18 @@ def is_locked(self): stmt = ( - "SELECT 1 FROM pg_locks" - " WHERE classid = %s AND objid = %s AND granted" + "SELECT 1 FROM pg_locks, pg_database" + " WHERE pg_locks.locktype = 'advisory'" + " AND pg_locks.classid = %s" + " AND pg_locks.objid = %s" + # objsubid is 2 when using the 2-argument version of the + # pg_advisory_* locking functions. + " AND pg_locks.objsubid = 2" + " AND pg_locks.granted" + # Advisory locks are local to each database so we join to + # pg_databases to discover the OID of the currrent database. + " AND pg_locks.database = pg_database.oid" + " AND pg_database.datname = current_database()" ) with closing(connection.cursor()) as cursor: cursor.execute(stmt, self) diff -Nru maas-1.5+bzr2252/src/maasserver/utils/tests/test_dblocks.py maas-1.5.4+bzr2294/src/maasserver/utils/tests/test_dblocks.py --- maas-1.5+bzr2252/src/maasserver/utils/tests/test_dblocks.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maasserver/utils/tests/test_dblocks.py 2014-09-03 14:18:31.000000000 +0000 @@ -16,7 +16,10 @@ from contextlib import closing -from django.db import connection +from django.db import ( + connection, + transaction, + ) from maasserver.utils import dblocks from maastesting.testcase import MAASTestCase @@ -40,6 +43,7 @@ lock = dblocks.DatabaseLock(self.getUniqueInteger()) self.assertEqual(lock, (lock.classid, lock.objid)) + @transaction.atomic def test_lock_actually_locked(self): objid = self.getUniqueInteger() lock = dblocks.DatabaseLock(objid) @@ -55,6 +59,7 @@ locks_released = locks_held - locks_held_after self.assertEqual({objid}, locks_released) + @transaction.atomic def test_is_locked(self): objid = self.getUniqueInteger() lock = dblocks.DatabaseLock(objid) @@ -64,6 +69,18 @@ self.assertTrue(lock.is_locked()) self.assertFalse(lock.is_locked()) + def test_obtaining_lock_fails_when_outside_of_transaction(self): + objid = self.getUniqueInteger() + lock = dblocks.DatabaseLock(objid) + self.assertRaises( + dblocks.DatabaseLockAttemptOutsideTransaction, + lock.__enter__) + + def test_releasing_lock_fails_when_lock_not_held(self): + objid = self.getUniqueInteger() + lock = dblocks.DatabaseLock(objid) + self.assertRaises(dblocks.DatabaseLockNotHeld, lock.__exit__) + def test_repr(self): lock = dblocks.DatabaseLock(self.getUniqueInteger()) self.assertEqual( diff -Nru maas-1.5+bzr2252/src/maastesting/factory.py maas-1.5.4+bzr2294/src/maastesting/factory.py --- maas-1.5+bzr2252/src/maastesting/factory.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/maastesting/factory.py 2014-09-03 14:18:31.000000000 +0000 @@ -35,6 +35,7 @@ from uuid import uuid1 from maastesting.fixtures import TempDirectory +import mock from netaddr import ( IPAddress, IPNetwork, @@ -264,6 +265,14 @@ return tarball + def make_streams(self, stdin=None, stdout=None, stderr=None): + """Make a fake return value for a SSHClient.exec_command.""" + # stdout.read() is called so stdout can't be None. + if stdout is None: + stdout = mock.Mock() + + return (stdin, stdout, stderr) + # Create factory singleton. factory = Factory() diff -Nru maas-1.5+bzr2252/src/metadataserver/address.py maas-1.5.4+bzr2294/src/metadataserver/address.py --- maas-1.5+bzr2252/src/metadataserver/address.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/metadataserver/address.py 2014-09-03 14:18:31.000000000 +0000 @@ -63,13 +63,13 @@ """ route_lines = list(ip_route_output) for line in route_lines: - match = re.match('default\s+.*\sdev\s+(\w+)', line) + match = re.match('default\s+.*\sdev\s+([^\s]+)', line) if match is not None: return match.groups()[0] # Still nothing? Try the first recognizable interface in the list. for line in route_lines: - match = re.match('\s*(?:\S+\s+)*dev\s+(\w+)', line) + match = re.match('\s*(?:\S+\s+)*dev\s+([^\s]+)', line) if match is not None: return match.groups()[0] return None diff -Nru maas-1.5+bzr2252/src/metadataserver/tests/test_address.py maas-1.5.4+bzr2294/src/metadataserver/tests/test_address.py --- maas-1.5+bzr2252/src/metadataserver/tests/test_address.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/metadataserver/tests/test_address.py 2014-09-03 14:18:31.000000000 +0000 @@ -59,6 +59,26 @@ self.assertEqual( 'eth1', address.find_default_interface(sample_ip_route)) + def test_find_default_interface_finds_default_tagged_interface(self): + sample_ip_route = [ + "default via 10.20.64.1 dev eth0.2", + "10.14.0.0/16 dev br0 proto kernel scope link src 10.14.4.1", + "10.90.90.0/24 dev br0 proto kernel scope link src 10.90.90.1", + "169.254.0.0/16 dev br0 scope link metric 1000", + ] + self.assertEqual( + 'eth0.2', address.find_default_interface(sample_ip_route)) + + def test_find_default_interface_finds_default_aliased_interface(self): + sample_ip_route = [ + "default via 10.20.64.1 dev eth0:2", + "10.14.0.0/16 dev br0 proto kernel scope link src 10.14.4.1", + "10.90.90.0/24 dev br0 proto kernel scope link src 10.90.90.1", + "169.254.0.0/16 dev br0 scope link metric 1000", + ] + self.assertEqual( + 'eth0:2', address.find_default_interface(sample_ip_route)) + def test_find_default_interface_makes_a_guess_if_no_default(self): sample_ip_route = [ "10.0.0.0/24 dev eth2 proto kernel scope link src 10.0.0.11 " @@ -69,6 +89,26 @@ self.assertEqual( 'eth2', address.find_default_interface(sample_ip_route)) + def test_find_default_tagged_interface_makes_a_guess_if_no_default(self): + sample_ip_route = [ + "10.0.0.0/24 dev eth2.4 proto kernel scope link src 10.0.0.11 " + "metric 2", + "10.1.0.0/24 dev virbr0 proto kernel scope link src 10.1.0.1", + "10.1.1.0/24 dev virbr1 proto kernel scope link src 10.1.1.1", + ] + self.assertEqual( + 'eth2.4', address.find_default_interface(sample_ip_route)) + + def test_find_default_aliased_interface_makes_a_guess_if_no_default(self): + sample_ip_route = [ + "10.0.0.0/24 dev eth2:4 proto kernel scope link src 10.0.0.11 " + "metric 2", + "10.1.0.0/24 dev virbr0 proto kernel scope link src 10.1.0.1", + "10.1.1.0/24 dev virbr1 proto kernel scope link src 10.1.1.1", + ] + self.assertEqual( + 'eth2:4', address.find_default_interface(sample_ip_route)) + def test_find_default_interface_returns_None_on_failure(self): self.assertIsNone(address.find_default_interface([])) diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/__init__.py maas-1.5.4+bzr2294/src/provisioningserver/boot/__init__.py --- maas-1.5+bzr2252/src/provisioningserver/boot/__init__.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/__init__.py 2014-09-03 14:18:31.000000000 +0000 @@ -23,6 +23,7 @@ abstractproperty, ) from errno import ENOENT +from io import BytesIO from os import path from provisioningserver.boot.tftppath import compose_image_path @@ -30,6 +31,23 @@ from provisioningserver.utils import locate_config from provisioningserver.utils.registry import Registry import tempita +from tftp.backend import IReader +from zope.interface import implementer + + +@implementer(IReader) +class BytesReader: + + def __init__(self, data): + super(BytesReader, self).__init__() + self.buffer = BytesIO(data) + self.size = len(data) + + def read(self, size): + return self.buffer.read(size) + + def finish(self): + self.buffer.close() class BootMethodError(Exception): @@ -81,6 +99,10 @@ __metaclass__ = ABCMeta + # Path prefix that is used for the pxelinux.cfg. Used for + # the dhcpd.conf that is generated. + path_prefix = None + @abstractproperty def name(self): """Name of the boot method.""" @@ -100,22 +122,24 @@ """ @abstractmethod - def match_config_path(self, path): - """Checks path for the configuration file that needs to be - generated. + def match_path(self, backend, path): + """Checks path for a file the boot method needs to handle. + :param backend: requesting backend :param path: requested path :returns: dict of match params from path, None if no match """ @abstractmethod - def render_config(self, kernel_params, **extra): - """Render a configuration file as a unicode string. + def get_reader(self, backend, kernel_params, **extra): + """Gets the reader the backend will use for this combination of + boot method, kernel parameters, and extra parameters. + :param backend: requesting backend :param kernel_params: An instance of `KernelParameters`. :param extra: Allow for other arguments. This is a safety valve; parameters generated in another component (for example, see - `TFTPBackend.get_config_reader`) won't cause this to break. + `TFTPBackend.get_boot_method_reader`) won't cause this to break. """ @abstractmethod @@ -202,11 +226,15 @@ # Import the supported boot methods after defining BootMethod. from provisioningserver.boot.pxe import PXEBootMethod from provisioningserver.boot.uefi import UEFIBootMethod +from provisioningserver.boot.powerkvm import PowerKVMBootMethod +from provisioningserver.boot.powernv import PowerNVBootMethod builtin_boot_methods = [ PXEBootMethod(), UEFIBootMethod(), + PowerKVMBootMethod(), + PowerNVBootMethod(), ] for method in builtin_boot_methods: BootMethodRegistry.register_item(method.name, method) diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/powerkvm.py maas-1.5.4+bzr2294/src/provisioningserver/boot/powerkvm.py --- maas-1.5+bzr2252/src/provisioningserver/boot/powerkvm.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/powerkvm.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,107 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""PowerKVM Boot Method""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [ + 'PowerKVMBootMethod', + ] + +import glob +import os.path +from textwrap import dedent + +from provisioningserver.boot import ( + BootMethod, + BootMethodInstallError, + utils, + ) +from provisioningserver.boot.install_bootloader import install_bootloader +from provisioningserver.utils import ( + call_and_check, + tempdir, + ) + + +GRUB_CONFIG = dedent("""\ + configfile (pxe)/grub/grub.cfg-${net_default_mac} + configfile (pxe)/grub/grub.cfg-default-ppc64el + """) + + +class PowerKVMBootMethod(BootMethod): + + name = "powerkvm" + template_subdir = None + bootloader_path = "bootppc64.bin" + arch_octet = "00:0C" + + def match_path(self, backend, path): + """Doesn't need to do anything, as the UEFIBootMethod provides + the grub implementation needed. + """ + return None + + def get_reader(self, backend, kernel_params, **extra): + """Doesn't need to do anything, as the UEFIBootMethod provides + the grub implementation needed. + """ + return None + + def install_bootloader(self, destination): + """Installs the required files for PowerKVM booting into the + tftproot. + """ + with tempdir() as tmp: + # Download the grub-ieee1275-bin package + data, filename = utils.get_updates_package( + 'grub-ieee1275-bin', 'http://ports.ubuntu.com', + 'main', 'ppc64el') + if data is None: + raise BootMethodInstallError( + 'Failed to download grub-ieee1275-bin package from ' + 'the archive.') + grub_output = os.path.join(tmp, filename) + with open(grub_output, 'wb') as stream: + stream.write(data) + + # Extract the package with dpkg, and install the shim + call_and_check(["dpkg", "-x", grub_output, tmp]) + + # Output the embedded config, so grub-mkimage can use it + config_output = os.path.join(tmp, 'grub.cfg') + with open(config_output, 'wb') as stream: + stream.write(GRUB_CONFIG.encode('utf-8')) + + # Get list of grub modules + module_dir = os.path.join( + tmp, 'usr', 'lib', 'grub', 'powerpc-ieee1275') + modules = [] + for module_path in glob.glob(os.path.join(module_dir, '*.mod')): + module_filename = os.path.basename(module_path) + module_name, _ = os.path.splitext(module_filename) + modules.append(module_name) + + # Generate the grub bootloader + mkimage_output = os.path.join(tmp, self.bootloader_path) + args = [ + 'grub-mkimage', + '-o', mkimage_output, + '-O', 'powerpc-ieee1275', + '-d', module_dir, + '-c', config_output, + ] + call_and_check(args + modules) + + install_bootloader( + mkimage_output, + os.path.join(destination, self.bootloader_path)) diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/powernv.py maas-1.5.4+bzr2294/src/provisioningserver/boot/powernv.py --- maas-1.5+bzr2252/src/provisioningserver/boot/powernv.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/powernv.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,158 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""PowerNV Boot Method""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [ + 'PowerNVBootMethod', + ] + +import re + +from provisioningserver.boot import ( + BootMethod, + BytesReader, + get_parameters, + ) +from provisioningserver.boot.pxe import ( + ARP_HTYPE, + re_mac_address, + ) +from provisioningserver.kernel_opts import compose_kernel_command_line +from provisioningserver.utils import find_mac_via_arp +from tftp.backend import FilesystemReader +from twisted.python.context import get + +# The pxelinux.cfg path is prefixed with the architecture for the +# PowerNV nodes. This prefix is set by the path-prefix dhcpd option. +# We assume that the ARP HTYPE (hardware type) that PXELINUX sends is +# always Ethernet. +re_config_file = r''' + # Optional leading slash(es). + ^/* + ppc64el # PowerNV pxe prefix, set by dhcpd + / + pxelinux[.]cfg # PXELINUX expects this. + / + (?: # either a MAC + {htype:02x} # ARP HTYPE. + - + (?P{re_mac_address.pattern}) # Capture MAC. + | # or "default" + default + ) + $ +''' + +re_config_file = re_config_file.format( + htype=ARP_HTYPE.ETHERNET, re_mac_address=re_mac_address) +re_config_file = re.compile(re_config_file, re.VERBOSE) + + +def format_bootif(mac): + """Formats a mac address into the BOOTIF format, expected by + the linux kernel.""" + mac = mac.replace(':', '-') + mac = mac.upper() + return '%02x-%s' % (ARP_HTYPE.ETHERNET, mac) + + +class PowerNVBootMethod(BootMethod): + + name = "powernv" + template_subdir = "pxe" + bootloader_path = "pxelinux.0" + arch_octet = "00:0E" + path_prefix = "ppc64el/" + + def get_remote_mac(self): + """Gets the requestors MAC address from arp cache. + + This is used, when the pxelinux.cfg is requested without the mac + address appended. This is needed to inject the BOOTIF into the + pxelinux.cfg that is returned to the node. + """ + remote_host, remote_port = get("remote", (None, None)) + return find_mac_via_arp(remote_host) + + def get_params(self, backend, path): + """Gets the matching parameters from the requested path.""" + match = re_config_file.match(path) + if match is not None: + return get_parameters(match) + if path.lstrip('/').startswith(self.path_prefix): + return {'path': path} + return None + + def match_path(self, backend, path): + """Checks path for the configuration file that needs to be + generated. + + :param backend: requesting backend + :param path: requested path + :returns: dict of match params from path, None if no match + """ + params = self.get_params(backend, path) + if params is None: + return None + params['arch'] = "ppc64el" + if 'mac' not in params: + mac = self.get_remote_mac() + if mac is not None: + params['mac'] = mac + return params + + def get_reader(self, backend, kernel_params, **extra): + """Render a configuration file as a unicode string. + + :param backend: requesting backend + :param kernel_params: An instance of `KernelParameters`. + :param extra: Allow for other arguments. This is a safety valve; + parameters generated in another component (for example, see + `TFTPBackend.get_config_reader`) won't cause this to break. + """ + # Due to the path prefix, all requested files from the client will + # contain that prefix. Removing the prefix from the path will return + # the correct path in the tftp root. + if 'path' in extra: + path = extra['path'] + path = path.replace(self.path_prefix, '', 1) + target_path = backend.base.descendant(path.split('/')) + return FilesystemReader(target_path) + + # Return empty config for PowerNV local. PowerNV fails to + # support the LOCALBOOT flag. Empty config will allow it + # to select the first device. + if kernel_params.purpose == 'local': + return BytesReader("".encode("utf-8")) + + template = self.get_template( + kernel_params.purpose, kernel_params.arch, + kernel_params.subarch) + namespace = self.compose_template_namespace(kernel_params) + + # Modify the kernel_command to inject the BOOTIF. PowerNV fails to + # support the IPAPPEND pxelinux flag. + def kernel_command(params): + cmd_line = compose_kernel_command_line(params) + if 'mac' in extra: + mac = extra['mac'] + mac = format_bootif(mac) + return '%s BOOTIF=%s' % (cmd_line, mac) + return cmd_line + + namespace['kernel_command'] = kernel_command + return BytesReader(template.substitute(namespace).encode("utf-8")) + + def install_bootloader(self, destination): + """Does nothing. No extra boot files are required. All of the boot + files from PXEBootMethod will suffice.""" diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/pxe.py maas-1.5.4+bzr2294/src/provisioningserver/boot/pxe.py --- maas-1.5+bzr2252/src/provisioningserver/boot/pxe.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/pxe.py 2014-09-03 14:18:31.000000000 +0000 @@ -22,6 +22,7 @@ from provisioningserver.boot import ( BootMethod, + BytesReader, get_parameters, ) from provisioningserver.boot.install_bootloader import install_bootloader @@ -78,10 +79,11 @@ bootloader_path = "pxelinux.0" arch_octet = "00:00" - def match_config_path(self, path): + def match_path(self, backend, path): """Checks path for the configuration file that needs to be generated. + :param backend: requesting backend :param path: requested path :returns: dict of match params from path, None if no match """ @@ -90,19 +92,20 @@ return None return get_parameters(match) - def render_config(self, kernel_params, **extra): + def get_reader(self, backend, kernel_params, **extra): """Render a configuration file as a unicode string. + :param backend: requesting backend :param kernel_params: An instance of `KernelParameters`. :param extra: Allow for other arguments. This is a safety valve; parameters generated in another component (for example, see - `TFTPBackend.get_config_reader`) won't cause this to break. + `TFTPBackend.get_boot_method_reader`) won't cause this to break. """ template = self.get_template( kernel_params.purpose, kernel_params.arch, kernel_params.subarch) namespace = self.compose_template_namespace(kernel_params) - return template.substitute(namespace) + return BytesReader(template.substitute(namespace).encode("utf-8")) def install_bootloader(self, destination): """Installs the required files for PXE booting into the diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_boot.py maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_boot.py --- maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_boot.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_boot.py 2014-09-03 14:18:31.000000000 +0000 @@ -24,6 +24,7 @@ from provisioningserver import boot from provisioningserver.boot import ( BootMethod, + BytesReader, gen_template_filenames, ) import tempita @@ -36,11 +37,11 @@ bootloader_path = "fake.efi" arch_octet = "00:00" - def match_config_path(self, path): + def match_path(self, backend, path): return {} - def render_config(kernel_params, **extra): - return "" + def get_reader(backend, kernel_params, **extra): + return BytesReader("") def install_bootloader(): pass diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_powerkvm.py maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_powerkvm.py --- maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_powerkvm.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_powerkvm.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,92 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Tests for `provisioningserver.boot.powerkvm`.""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [] + +from contextlib import contextmanager +import os + +from maastesting.factory import factory +from maastesting.matchers import MockCalledOnceWith +from maastesting.testcase import MAASTestCase +from provisioningserver.boot import ( + BootMethodInstallError, + powerkvm as powerkvm_module, + utils, + ) +from provisioningserver.boot.powerkvm import ( + GRUB_CONFIG, + PowerKVMBootMethod, + ) +from provisioningserver.tests.test_kernel_opts import make_kernel_parameters + + +class TestPowerKVMBootMethod(MAASTestCase): + """Tests `provisioningserver.boot.powerkvm.PowerKVMBootMethod`.""" + + def test_match_path_returns_None(self): + method = PowerKVMBootMethod() + paths = [factory.getRandomString() for _ in range(3)] + for path in paths: + self.assertEqual(None, method.match_path(None, path)) + + def test_get_reader_returns_None(self): + method = PowerKVMBootMethod() + params = [make_kernel_parameters() for _ in range(3)] + for param in params: + self.assertEqual(None, method.get_reader(None, params)) + + def test_install_bootloader_get_package_raises_error(self): + method = PowerKVMBootMethod() + self.patch(utils, 'get_updates_package').return_value = (None, None) + self.assertRaises( + BootMethodInstallError, method.install_bootloader, None) + + def test_install_bootloader(self): + method = PowerKVMBootMethod() + filename = factory.make_name('dpkg') + data = factory.getRandomString() + tmp = self.make_dir() + dest = self.make_dir() + + @contextmanager + def tempdir(): + try: + yield tmp + finally: + pass + + mock_get_updates_package = self.patch(utils, 'get_updates_package') + mock_get_updates_package.return_value = (data, filename) + self.patch(powerkvm_module, 'call_and_check') + self.patch(powerkvm_module, 'tempdir').side_effect = tempdir + + mock_install_bootloader = self.patch( + powerkvm_module, 'install_bootloader') + + method.install_bootloader(dest) + + with open(os.path.join(tmp, filename), 'rb') as stream: + saved_data = stream.read() + self.assertEqual(data, saved_data) + + with open(os.path.join(tmp, 'grub.cfg'), 'rb') as stream: + saved_config = stream.read().decode('utf-8') + self.assertEqual(GRUB_CONFIG, saved_config) + + mkimage_expected = os.path.join(tmp, method.bootloader_path) + dest_expected = os.path.join(dest, method.bootloader_path) + self.assertThat( + mock_install_bootloader, + MockCalledOnceWith(mkimage_expected, dest_expected)) diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_powernv.py maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_powernv.py --- maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_powernv.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_powernv.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,337 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Tests for `provisioningserver.boot.powernv`.""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [] + +import os +import re + +from maastesting.factory import factory +from maastesting.testcase import MAASTestCase +from provisioningserver.boot import BytesReader +from provisioningserver.boot.powernv import ( + ARP_HTYPE, + format_bootif, + PowerNVBootMethod, + re_config_file, + ) +from provisioningserver.boot.tests.test_pxe import parse_pxe_config +from provisioningserver.boot.tftppath import compose_image_path +from provisioningserver.testing.config import set_tftp_root +from provisioningserver.tests.test_kernel_opts import make_kernel_parameters +from provisioningserver.tftp import TFTPBackend +from testtools.matchers import ( + IsInstance, + MatchesAll, + MatchesRegex, + Not, + StartsWith, + ) + + +def compose_config_path(mac): + """Compose the TFTP path for a PowerNV PXE configuration file. + + The path returned is relative to the TFTP root, as it would be + identified by clients on the network. + + :param mac: A MAC address, in IEEE 802 hyphen-separated form, + corresponding to the machine for which this configuration is + relevant. This relates to PXELINUX's lookup protocol. + :return: Path for the corresponding PXE config file as exposed over + TFTP. + """ + # Not using os.path.join: this is a TFTP path, not a native path. Yes, in + # practice for us they're the same. We always assume that the ARP HTYPE + # (hardware type) that PXELINUX sends is Ethernet. + return "ppc64el/pxelinux.cfg/{htype:02x}-{mac}".format( + htype=ARP_HTYPE.ETHERNET, mac=mac) + + +def get_example_path_and_components(): + """Return a plausible path and its components. + + The path is intended to match `re_config_file`, and the components are + the expected groups from a match. + """ + components = {"mac": factory.getRandomMACAddress("-")} + config_path = compose_config_path(components["mac"]) + return config_path, components + + +class TestPowerNVBootMethod(MAASTestCase): + + def make_tftp_root(self): + """Set, and return, a temporary TFTP root directory.""" + tftproot = self.make_dir() + self.useFixture(set_tftp_root(tftproot)) + return tftproot + + def test_compose_config_path_follows_maas_pxe_directory_layout(self): + name = factory.make_name('config') + self.assertEqual( + 'ppc64el/pxelinux.cfg/%02x-%s' % (ARP_HTYPE.ETHERNET, name), + compose_config_path(name)) + + def test_compose_config_path_does_not_include_tftp_root(self): + tftproot = self.make_tftp_root() + name = factory.make_name('config') + self.assertThat( + compose_config_path(name), + Not(StartsWith(tftproot))) + + def test_bootloader_path(self): + method = PowerNVBootMethod() + self.assertEqual('pxelinux.0', method.bootloader_path) + + def test_bootloader_path_does_not_include_tftp_root(self): + tftproot = self.make_tftp_root() + method = PowerNVBootMethod() + self.assertThat( + method.bootloader_path, + Not(StartsWith(tftproot))) + + def test_name(self): + method = PowerNVBootMethod() + self.assertEqual('powernv', method.name) + + def test_template_subdir(self): + method = PowerNVBootMethod() + self.assertEqual('pxe', method.template_subdir) + + def test_arch_octet(self): + method = PowerNVBootMethod() + self.assertEqual('00:0E', method.arch_octet) + + def test_path_prefix(self): + method = PowerNVBootMethod() + self.assertEqual('ppc64el/', method.path_prefix) + + +class TestPowerNVBootMethodMatchPath(MAASTestCase): + """Tests for + `provisioningserver.boot.powernv.PowerNVBootMethod.match_path`. + """ + + def test_match_path_pxe_config_with_mac(self): + method = PowerNVBootMethod() + config_path, expected = get_example_path_and_components() + params = method.match_path(None, config_path) + expected['arch'] = 'ppc64el' + self.assertEqual(expected, params) + + def test_match_path_pxe_config_without_mac(self): + method = PowerNVBootMethod() + fake_mac = factory.getRandomMACAddress() + self.patch(method, 'get_remote_mac').return_value = fake_mac + config_path = 'ppc64el/pxelinux.cfg/default' + params = method.match_path(None, config_path) + expected = { + 'arch': 'ppc64el', + 'mac': fake_mac, + } + self.assertEqual(expected, params) + + def test_match_path_pxe_prefix_request(self): + method = PowerNVBootMethod() + fake_mac = factory.getRandomMACAddress() + self.patch(method, 'get_remote_mac').return_value = fake_mac + file_path = 'ppc64el/file' + params = method.match_path(None, file_path) + expected = { + 'arch': 'ppc64el', + 'mac': fake_mac, + 'path': file_path, + } + self.assertEqual(expected, params) + + +class TestPowerNVBootMethodRenderConfig(MAASTestCase): + """Tests for + `provisioningserver.boot.powernv.PowerNVBootMethod.get_reader` + """ + + def test_get_reader_install(self): + # Given the right configuration options, the PXE configuration is + # correctly rendered. + method = PowerNVBootMethod() + params = make_kernel_parameters(self, purpose="install") + output = method.get_reader(backend=None, kernel_params=params) + # The output is a BytesReader. + self.assertThat(output, IsInstance(BytesReader)) + output = output.read(10000) + # The template has rendered without error. PXELINUX configurations + # typically start with a DEFAULT line. + self.assertThat(output, StartsWith("DEFAULT ")) + # The PXE parameters are all set according to the options. + image_dir = compose_image_path( + arch=params.arch, subarch=params.subarch, + release=params.release, label=params.label) + self.assertThat( + output, MatchesAll( + MatchesRegex( + r'.*^\s+KERNEL %s/di-kernel$' % re.escape(image_dir), + re.MULTILINE | re.DOTALL), + MatchesRegex( + r'.*^\s+INITRD %s/di-initrd$' % re.escape(image_dir), + re.MULTILINE | re.DOTALL), + MatchesRegex( + r'.*^\s+APPEND .+?$', + re.MULTILINE | re.DOTALL))) + + def test_get_reader_with_extra_arguments_does_not_affect_output(self): + # get_reader() allows any keyword arguments as a safety valve. + method = PowerNVBootMethod() + options = { + "backend": None, + "kernel_params": make_kernel_parameters(self, purpose="install"), + } + # Capture the output before sprinking in some random options. + output_before = method.get_reader(**options).read(10000) + # Sprinkle some magic in. + options.update( + (factory.make_name("name"), factory.make_name("value")) + for _ in range(10)) + # Capture the output after sprinking in some random options. + output_after = method.get_reader(**options).read(10000) + # The generated template is the same. + self.assertEqual(output_before, output_after) + + def test_get_reader_with_local_purpose(self): + # If purpose is "local", output should be empty string. + method = PowerNVBootMethod() + options = { + "backend": None, + "kernel_params": make_kernel_parameters(purpose="local"), + } + output = method.get_reader(**options).read(10000) + self.assertIn("", output) + + def test_get_reader_appends_bootif(self): + method = PowerNVBootMethod() + fake_mac = factory.getRandomMACAddress() + params = make_kernel_parameters(self, purpose="install") + output = method.get_reader( + backend=None, kernel_params=params, arch='ppc64el', mac=fake_mac) + output = output.read(10000) + config = parse_pxe_config(output) + expected = 'BOOTIF=%s' % format_bootif(fake_mac) + self.assertIn(expected, config['execute']['APPEND']) + + +class TestPowerNVBootMethodPathPrefix(MAASTestCase): + """Tests for + `provisioningserver.boot.powernv.PowerNVBootMethod.get_reader`. + """ + + def test_get_reader_path_prefix(self): + data = factory.getRandomString().encode("ascii") + temp_file = self.make_file(name="example", contents=data) + temp_dir = os.path.dirname(temp_file) + backend = TFTPBackend(temp_dir, "http://nowhere.example.com/") + method = PowerNVBootMethod() + options = { + 'backend': backend, + 'kernel_params': make_kernel_parameters(), + 'path': 'ppc64el/example', + } + reader = method.get_reader(**options) + self.addCleanup(reader.finish) + self.assertEqual(len(data), reader.size) + self.assertEqual(data, reader.read(len(data))) + self.assertEqual(b"", reader.read(1)) + + def test_get_reader_path_prefix_only_removes_first_occurrence(self): + data = factory.getRandomString().encode("ascii") + temp_dir = self.make_dir() + temp_subdir = os.path.join(temp_dir, 'ppc64el') + os.mkdir(temp_subdir) + factory.make_file(temp_subdir, "example", data) + backend = TFTPBackend(temp_dir, "http://nowhere.example.com/") + method = PowerNVBootMethod() + options = { + 'backend': backend, + 'kernel_params': make_kernel_parameters(), + 'path': 'ppc64el/ppc64el/example', + } + reader = method.get_reader(**options) + self.addCleanup(reader.finish) + self.assertEqual(len(data), reader.size) + self.assertEqual(data, reader.read(len(data))) + self.assertEqual(b"", reader.read(1)) + + +class TestPowerNVBootMethodRegex(MAASTestCase): + """Tests for + `provisioningserver.boot.powernv.PowerNVBootMethod.re_config_file`. + """ + + def test_re_config_file_is_compatible_with_config_path_generator(self): + # The regular expression for extracting components of the file path is + # compatible with the PXE config path generator. + for iteration in range(10): + config_path, args = get_example_path_and_components() + match = re_config_file.match(config_path) + self.assertIsNotNone(match, config_path) + self.assertEqual(args, match.groupdict()) + + def test_re_config_file_with_leading_slash(self): + # The regular expression for extracting components of the file path + # doesn't care if there's a leading forward slash; the TFTP server is + # easy on this point, so it makes sense to be also. + config_path, args = get_example_path_and_components() + # Ensure there's a leading slash. + config_path = "/" + config_path.lstrip("/") + match = re_config_file.match(config_path) + self.assertIsNotNone(match, config_path) + self.assertEqual(args, match.groupdict()) + + def test_re_config_file_without_leading_slash(self): + # The regular expression for extracting components of the file path + # doesn't care if there's no leading forward slash; the TFTP server is + # easy on this point, so it makes sense to be also. + config_path, args = get_example_path_and_components() + # Ensure there's no leading slash. + config_path = config_path.lstrip("/") + match = re_config_file.match(config_path) + self.assertIsNotNone(match, config_path) + self.assertEqual(args, match.groupdict()) + + def test_re_config_file_matches_classic_pxelinux_cfg(self): + # The default config path is simply "pxelinux.cfg" (without + # leading slash). The regex matches this. + mac = 'aa-bb-cc-dd-ee-ff' + match = re_config_file.match('ppc64el/pxelinux.cfg/01-%s' % mac) + self.assertIsNotNone(match) + self.assertEqual({'mac': mac}, match.groupdict()) + + def test_re_config_file_matches_pxelinux_cfg_with_leading_slash(self): + mac = 'aa-bb-cc-dd-ee-ff' + match = re_config_file.match('/ppc64el/pxelinux.cfg/01-%s' % mac) + self.assertIsNotNone(match) + self.assertEqual({'mac': mac}, match.groupdict()) + + def test_re_config_file_does_not_match_non_config_file(self): + self.assertIsNone(re_config_file.match('ppc64el/pxelinux.cfg/kernel')) + + def test_re_config_file_does_not_match_file_in_root(self): + self.assertIsNone(re_config_file.match('01-aa-bb-cc-dd-ee-ff')) + + def test_re_config_file_does_not_match_file_not_in_pxelinux_cfg(self): + self.assertIsNone(re_config_file.match('foo/01-aa-bb-cc-dd-ee-ff')) + + def test_re_config_file_with_default(self): + match = re_config_file.match('ppc64el/pxelinux.cfg/default') + self.assertIsNotNone(match) + self.assertEqual({'mac': None}, match.groupdict()) diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_pxe.py maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_pxe.py --- maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_pxe.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_pxe.py 2014-09-03 14:18:31.000000000 +0000 @@ -20,6 +20,7 @@ from maastesting.factory import factory from maastesting.testcase import MAASTestCase from provisioningserver import kernel_opts +from provisioningserver.boot import BytesReader from provisioningserver.boot.pxe import ( ARP_HTYPE, PXEBootMethod, @@ -150,14 +151,15 @@ class TestPXEBootMethodRenderConfig(MAASTestCase): """Tests for `provisioningserver.boot.pxe.PXEBootMethod.render_config`.""" - def test_render_install(self): + def test_get_reader_install(self): # Given the right configuration options, the PXE configuration is # correctly rendered. method = PXEBootMethod() params = make_kernel_parameters(self, purpose="install") - output = method.render_config(kernel_params=params) - # The output is always a Unicode string. - self.assertThat(output, IsInstance(unicode)) + output = method.get_reader(backend=None, kernel_params=params) + # The output is a BytesReader. + self.assertThat(output, IsInstance(BytesReader)) + output = output.read(10000) # The template has rendered without error. PXELINUX configurations # typically start with a DEFAULT line. self.assertThat(output, StartsWith("DEFAULT ")) @@ -177,54 +179,58 @@ r'.*^\s+APPEND .+?$', re.MULTILINE | re.DOTALL))) - def test_render_with_extra_arguments_does_not_affect_output(self): - # render_config() allows any keyword arguments as a safety valve. + def test_get_reader_with_extra_arguments_does_not_affect_output(self): + # get_reader() allows any keyword arguments as a safety valve. method = PXEBootMethod() options = { + "backend": None, "kernel_params": make_kernel_parameters(self, purpose="install"), } # Capture the output before sprinking in some random options. - output_before = method.render_config(**options) + output_before = method.get_reader(**options).read(10000) # Sprinkle some magic in. options.update( (factory.make_name("name"), factory.make_name("value")) for _ in range(10)) # Capture the output after sprinking in some random options. - output_after = method.render_config(**options) + output_after = method.get_reader(**options).read(10000) # The generated template is the same. self.assertEqual(output_before, output_after) - def test_render_config_with_local_purpose(self): + def test_get_reader_with_local_purpose(self): # If purpose is "local", the config.localboot.template should be # used. method = PXEBootMethod() options = { + "backend": None, "kernel_params": make_kernel_parameters(purpose="local"), } - output = method.render_config(**options) + output = method.get_reader(**options).read(10000) self.assertIn("LOCALBOOT 0", output) - def test_render_config_with_local_purpose_i386_arch(self): + def test_get_reader_with_local_purpose_i386_arch(self): # Intel i386 is a special case and needs to use the chain.c32 # loader as the LOCALBOOT PXE directive is unreliable. method = PXEBootMethod() options = { + "backend": None, "kernel_params": make_kernel_parameters( arch="i386", purpose="local"), } - output = method.render_config(**options) + output = method.get_reader(**options).read(10000) self.assertIn("chain.c32", output) self.assertNotIn("LOCALBOOT", output) - def test_render_config_with_local_purpose_amd64_arch(self): + def test_get_reader_with_local_purpose_amd64_arch(self): # Intel amd64 is a special case and needs to use the chain.c32 # loader as the LOCALBOOT PXE directive is unreliable. method = PXEBootMethod() options = { + "backend": None, "kernel_params": make_kernel_parameters( arch="amd64", purpose="local"), } - output = method.render_config(**options) + output = method.get_reader(**options).read(10000) self.assertIn("chain.c32", output) self.assertNotIn("LOCALBOOT", output) @@ -237,17 +243,18 @@ ("xinstall", dict(purpose="xinstall")), ] - def test_render_config_scenarios(self): + def test_get_reader_scenarios(self): # The commissioning config uses an extra PXELINUX module to auto # select between i386 and amd64. method = PXEBootMethod() get_ephemeral_name = self.patch(kernel_opts, "get_ephemeral_name") get_ephemeral_name.return_value = factory.make_name("ephemeral") options = { + "backend": None, "kernel_params": make_kernel_parameters( testcase=self, subarch="generic", purpose=self.purpose), } - output = method.render_config(**options) + output = method.get_reader(**options).read(10000) config = parse_pxe_config(output) # The default section is defined. default_section_label = config.header["DEFAULT"] diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_uefi.py maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_uefi.py --- maas-1.5+bzr2252/src/provisioningserver/boot/tests/test_uefi.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/tests/test_uefi.py 2014-09-03 14:18:31.000000000 +0000 @@ -18,6 +18,7 @@ from maastesting.factory import factory from maastesting.testcase import MAASTestCase +from provisioningserver.boot import BytesReader from provisioningserver.boot.tftppath import compose_image_path from provisioningserver.boot.uefi import ( re_config_file, @@ -60,14 +61,15 @@ class TestRenderUEFIConfig(MAASTestCase): """Tests for `provisioningserver.boot.uefi.UEFIBootMethod`.""" - def test_render(self): + def test_get_reader(self): # Given the right configuration options, the UEFI configuration is # correctly rendered. method = UEFIBootMethod() params = make_kernel_parameters(purpose="install") - output = method.render_config(kernel_params=params) - # The output is always a Unicode string. - self.assertThat(output, IsInstance(unicode)) + output = method.get_reader(backend=None, kernel_params=params) + # The output is a BytesReader. + self.assertThat(output, IsInstance(BytesReader)) + output = output.read(10000) # The template has rendered without error. UEFI configurations # typically start with a DEFAULT line. self.assertThat(output, StartsWith("set default=\"0\"")) @@ -85,31 +87,34 @@ r'.*^\s+initrd %s/di-initrd$' % re.escape(image_dir), re.MULTILINE | re.DOTALL))) - def test_render_with_extra_arguments_does_not_affect_output(self): - # render_config() allows any keyword arguments as a safety valve. + def test_get_reader_with_extra_arguments_does_not_affect_output(self): + # get_reader() allows any keyword arguments as a safety valve. method = UEFIBootMethod() options = { + "backend": None, "kernel_params": make_kernel_parameters(purpose="install"), } # Capture the output before sprinking in some random options. - output_before = method.render_config(**options) + output_before = method.get_reader(**options).read(10000) # Sprinkle some magic in. options.update( (factory.make_name("name"), factory.make_name("value")) for _ in range(10)) # Capture the output after sprinking in some random options. - output_after = method.render_config(**options) + output_after = method.get_reader(**options).read(10000) # The generated template is the same. self.assertEqual(output_before, output_after) - def test_render_config_with_local_purpose(self): + def test_get_reader_with_local_purpose(self): # If purpose is "local", the config.localboot.template should be # used. method = UEFIBootMethod() options = { - "kernel_params": make_kernel_parameters(purpose="local"), + "backend": None, + "kernel_params": make_kernel_parameters( + purpose="local", arch="amd64"), } - output = method.render_config(**options) + output = method.get_reader(**options).read(10000) self.assertIn("configfile /efi/ubuntu/grub.cfg", output) diff -Nru maas-1.5+bzr2252/src/provisioningserver/boot/uefi.py maas-1.5.4+bzr2294/src/provisioningserver/boot/uefi.py --- maas-1.5+bzr2252/src/provisioningserver/boot/uefi.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/boot/uefi.py 2014-09-03 14:18:31.000000000 +0000 @@ -25,6 +25,7 @@ from provisioningserver.boot import ( BootMethod, BootMethodInstallError, + BytesReader, get_parameters, utils, ) @@ -120,10 +121,11 @@ bootloader_path = "bootx64.efi" arch_octet = "00:07" # AMD64 EFI - def match_config_path(self, path): + def match_path(self, backend, path): """Checks path for the configuration file that needs to be generated. + :param backend: requesting backend :param path: requested path :returns: dict of match params from path, None if no match """ @@ -139,19 +141,20 @@ return params - def render_config(self, kernel_params, **extra): + def get_reader(self, backend, kernel_params, **extra): """Render a configuration file as a unicode string. + :param backend: requesting backend :param kernel_params: An instance of `KernelParameters`. :param extra: Allow for other arguments. This is a safety valve; parameters generated in another component (for example, see - `TFTPBackend.get_config_reader`) won't cause this to break. + `TFTPBackend.get_boot_method_reader`) won't cause this to break. """ template = self.get_template( kernel_params.purpose, kernel_params.arch, kernel_params.subarch) namespace = self.compose_template_namespace(kernel_params) - return template.substitute(namespace) + return BytesReader(template.substitute(namespace).encode("utf-8")) def install_bootloader(self, destination): """Installs the required files for UEFI booting into the diff -Nru maas-1.5+bzr2252/src/provisioningserver/custom_hardware/tests/test_ucsm.py maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/tests/test_ucsm.py --- maas-1.5+bzr2252/src/provisioningserver/custom_hardware/tests/test_ucsm.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/tests/test_ucsm.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,638 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Tests for ``provisioningserver.custom_hardware.ucsm``.""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [] + +from itertools import permutations +import random +from StringIO import StringIO +import urllib2 + +from lxml.etree import ( + Element, + SubElement, + XML, + ) +from maastesting.factory import factory +from maastesting.matchers import ( + MockCalledOnceWith, + MockCallsMatch, + MockNotCalled, + ) +from maastesting.testcase import MAASTestCase +from mock import ( + ANY, + call, + Mock, + ) +from provisioningserver.custom_hardware import ( + ucsm, + utils, + ) +from provisioningserver.custom_hardware.ucsm import ( + get_children, + get_first_booter, + get_macs, + get_power_command, + get_server_power_control, + get_servers, + get_service_profile, + logged_in, + make_policy_change, + make_request_data, + parse_response, + power_control_ucsm, + probe_and_enlist_ucsm, + probe_servers, + RO_KEYS, + set_lan_boot_default, + set_server_power_control, + strip_ro_keys, + UCSM_XML_API, + UCSM_XML_API_Error, + ) + + +def make_api(url='http://url', user='u', password='p', + cookie='foo', mock_call=True): + api = UCSM_XML_API(url, user, password) + api.cookie = cookie + return api + + +def make_api_patch_call(testcase, *args, **kwargs): + api = make_api(*args, **kwargs) + mock = testcase.patch(api, '_call') + return api, mock + + +def make_fake_result(root_class, child_tag, container='outConfigs'): + fake_result = Element(root_class) + outConfigs = SubElement(fake_result, container) + outConfigs.append(Element(child_tag)) + return outConfigs + + +def make_class(): + return factory.make_name('class') + + +def make_dn(): + return factory.make_name('dn') + + +def make_server(): + return factory.make_name('server') + + +class TestUCSMXMLAPIError(MAASTestCase): + """Tests for ``UCSM_XML_API_Error``.""" + + def test_includes_code_and_msg(self): + def raise_error(): + raise UCSM_XML_API_Error('bad', 4224) + + error = self.assertRaises(UCSM_XML_API_Error, raise_error) + + self.assertEqual('bad', error.args[0]) + self.assertEqual(4224, error.code) + + +class TestMakeRequestData(MAASTestCase): + """Tests for ``make_request_data``.""" + + def test_no_children(self): + fields = {'hello': 'there'} + request_data = make_request_data('foo', fields) + root = XML(request_data) + self.assertEqual('foo', root.tag) + self.assertEqual('there', root.get('hello')) + + def test_with_children(self): + fields = {'hello': 'there'} + children_tags = ['bar', 'baz'] + children = [Element(child_tag) for child_tag in children_tags] + request_data = make_request_data('foo', fields, children) + root = XML(request_data) + self.assertEqual('foo', root.tag) + self.assertItemsEqual(children_tags, (e.tag for e in root)) + + def test_no_fields(self): + request_data = make_request_data('foo') + root = XML(request_data) + self.assertEqual('foo', root.tag) + + +class TestParseResonse(MAASTestCase): + """Tests for ``parse_response``.""" + + def test_no_error(self): + xml = '' + response = parse_response(xml) + self.assertEqual('foo', response.tag) + + def test_error(self): + xml = '' + self.assertRaises(UCSM_XML_API_Error, parse_response, xml) + + +class TestLogin(MAASTestCase): + """"Tests for ``UCSM_XML_API.login``.""" + + def test_login_assigns_cookie(self): + cookie = 'chocolate chip' + api, mock = make_api_patch_call(self) + mock.return_value = Element('aaaLogin', {'outCookie': cookie}) + api.login() + self.assertEqual(cookie, api.cookie) + + def test_login_call_parameters(self): + user = 'user' + password = 'pass' + api, mock = make_api_patch_call(self, user=user, password=password) + api.login() + fields = {'inName': user, 'inPassword': password} + self.assertThat(mock, MockCalledOnceWith('aaaLogin', fields)) + + +class TestLogout(MAASTestCase): + """"Tests for ``UCSM_XML_API.logout``.""" + + def test_logout_clears_cookie(self): + api = make_api() + self.patch(api, '_call') + api.logout() + self.assertIsNone(api.cookie) + + def test_logout_uses_cookie(self): + api, mock = make_api_patch_call(self) + cookie = api.cookie + api.logout() + fields = {'inCookie': cookie} + self.assertThat(mock, MockCalledOnceWith('aaaLogout', fields)) + + +class TestConfigResolveClass(MAASTestCase): + """"Tests for ``UCSM_XML_API.config_resolve_class``.""" + + def test_no_filters(self): + class_id = make_class() + api, mock = make_api_patch_call(self) + api.config_resolve_class(class_id) + fields = {'cookie': api.cookie, 'classId': class_id} + self.assertThat(mock, MockCalledOnceWith('configResolveClass', fields, + ANY)) + + def test_with_filters(self): + class_id = make_class() + filter_element = Element('hi') + api, mock = make_api_patch_call(self) + api.config_resolve_class(class_id, [filter_element]) + in_filters = mock.call_args[0][2] + self.assertEqual([filter_element], in_filters[0][:]) + + def test_return_response(self): + api, mock = make_api_patch_call(self) + mock.return_value = Element('test') + result = api.config_resolve_class('c') + self.assertEqual(mock.return_value, result) + + +class TestConfigResolveChildren(MAASTestCase): + """"Tests for ``UCSM_XML_API.config_resolve_children``.""" + + def test_parameters(self): + dn = make_dn() + class_id = make_class() + api, mock = make_api_patch_call(self) + api.config_resolve_children(dn, class_id) + fields = {'inDn': dn, 'classId': class_id, 'cookie': api.cookie} + self.assertThat(mock, + MockCalledOnceWith('configResolveChildren', fields)) + + def test_no_class_id(self): + dn = make_dn() + api, mock = make_api_patch_call(self) + api.config_resolve_children(dn) + fields = {'inDn': dn, 'cookie': api.cookie} + self.assertThat(mock, + MockCalledOnceWith('configResolveChildren', fields)) + + def test_return_response(self): + api, mock = make_api_patch_call(self) + mock.return_value = Element('test') + result = api.config_resolve_children('d', 'c') + self.assertEqual(mock.return_value, result) + + +class TestConfigConfMo(MAASTestCase): + """"Tests for ``UCSM_XML_API.config_conf_mo``.""" + + def test_parameters(self): + dn = make_dn() + config_items = [Element('hi')] + api, mock = make_api_patch_call(self) + api.config_conf_mo(dn, config_items) + fields = {'dn': dn, 'cookie': api.cookie} + self.assertThat(mock, MockCalledOnceWith('configConfMo', fields, ANY)) + in_configs = mock.call_args[0][2] + self.assertEqual(config_items, in_configs[0][:]) + + +class TestCall(MAASTestCase): + """"Tests for ``UCSM_XML_API._call``.""" + + def test_call(self): + name = 'method' + fields = {1: 2} + children = [3, 4] + request = '' + response = Element('good') + api = make_api() + + mock_make_request_data = self.patch(ucsm, 'make_request_data') + mock_make_request_data.return_value = request + + mock_send_request = self.patch(api, '_send_request') + mock_send_request.return_value = response + + api._call(name, fields, children) + self.assertThat(mock_make_request_data, + MockCalledOnceWith(name, fields, children)) + self.assertThat(mock_send_request, MockCalledOnceWith(request)) + + +class TestSendRequest(MAASTestCase): + """"Tests for ``UCSM_XML_API._send_request``.""" + + def test_send_request(self): + request_data = 'foo' + api = make_api() + self.patch(api, '_call') + stream = StringIO('') + mock = self.patch(urllib2, 'urlopen') + mock.return_value = stream + response = api._send_request(request_data) + self.assertEqual('hi', response.tag) + urllib_request = mock.call_args[0][0] + self.assertEqual(request_data, urllib_request.data) + + +class TestConfigResolveDn(MAASTestCase): + """Tests for ``UCSM_XML_API.config_resolve_dn``.""" + + def test_parameters(self): + api, mock = make_api_patch_call(self) + test_dn = make_dn() + fields = {'cookie': api.cookie, 'dn': test_dn} + api.config_resolve_dn(test_dn) + self.assertThat(mock, + MockCalledOnceWith('configResolveDn', fields)) + + +class TestGetServers(MAASTestCase): + """Tests for ``get_servers``.""" + + def test_uses_uuid(self): + uuid = factory.getRandomUUID() + api = make_api() + mock = self.patch(api, 'config_resolve_class') + get_servers(api, uuid) + filters = mock.call_args[0][1] + attrib = {'class': 'computeItem', 'property': 'uuid', 'value': uuid} + self.assertEqual(attrib, filters[0].attrib) + + def test_returns_result(self): + uuid = factory.getRandomUUID() + api = make_api() + fake_result = make_fake_result('configResolveClass', 'found') + self.patch(api, 'config_resolve_class').return_value = fake_result + result = get_servers(api, uuid) + self.assertEqual('found', result[0].tag) + + def test_class_id(self): + uuid = factory.getRandomUUID() + api = make_api() + mock = self.patch(api, 'config_resolve_class') + get_servers(api, uuid) + self.assertThat(mock, MockCalledOnceWith('computeItem', ANY)) + + +class TestGetChildren(MAASTestCase): + """Tests for ``get_children``.""" + + def test_returns_result(self): + search_class = make_class() + api = make_api() + fake_result = make_fake_result('configResolveChildren', search_class) + self.patch(api, 'config_resolve_children').return_value = fake_result + in_element = Element('test', {'dn': make_dn()}) + class_id = search_class + result = get_children(api, in_element, class_id) + self.assertEqual(search_class, result[0].tag) + + def test_parameters(self): + search_class = make_class() + parent_dn = make_dn() + api = make_api() + mock = self.patch(api, 'config_resolve_children') + in_element = Element('test', {'dn': parent_dn}) + class_id = search_class + get_children(api, in_element, class_id) + self.assertThat(mock, MockCalledOnceWith(parent_dn, search_class)) + + +class TestGetMacs(MAASTestCase): + """Tests for ``get_macs``.""" + + def test_gets_adaptors(self): + adaptor = 'adaptor' + server = make_server() + mac = 'xx' + api = make_api() + mock = self.patch(ucsm, 'get_children') + + def fake_get_children(api, element, class_id): + if class_id == 'adaptorUnit': + return [adaptor] + elif class_id == 'adaptorHostEthIf': + return [Element('ethif', {'mac': mac})] + + mock.side_effect = fake_get_children + macs = get_macs(api, server) + self.assertThat(mock, MockCallsMatch( + call(api, server, 'adaptorUnit'), + call(api, adaptor, 'adaptorHostEthIf'))) + self.assertEqual([mac], macs) + + +class TestProbeServers(MAASTestCase): + """Tests for ``probe_servers``.""" + + def test_uses_api(self): + api = make_api() + mock = self.patch(ucsm, 'get_servers') + probe_servers(api) + self.assertThat(mock, MockCalledOnceWith(api)) + + def test_returns_results(self): + servers = [{'uuid': factory.getRandomUUID()}] + mac = 'mac' + api = make_api() + self.patch(ucsm, 'get_servers').return_value = servers + self.patch(ucsm, 'get_macs').return_value = [mac] + server_list = probe_servers(api) + self.assertEqual([(servers[0], [mac])], server_list) + + +class TestGetServerPowerControl(MAASTestCase): + """Tests for ``get_server_power_control``.""" + + def test_get_server_power_control(self): + api = make_api() + mock = self.patch(api, 'config_resolve_children') + fake_result = make_fake_result('configResolveChildren', 'lsPower') + mock.return_value = fake_result + dn = make_dn() + server = Element('computeItem', {'assignedToDn': dn}) + power_control = get_server_power_control(api, server) + self.assertThat(mock, MockCalledOnceWith(dn, 'lsPower')) + self.assertEqual('lsPower', power_control.tag) + + +class TestSetServerPowerControl(MAASTestCase): + """Tests for ``set_server_power_control``.""" + + def test_set_server_power_control(self): + api = make_api() + power_dn = make_dn() + power_control = Element('lsPower', {'dn': power_dn}) + config_conf_mo_mock = self.patch(api, 'config_conf_mo') + state = 'state' + set_server_power_control(api, power_control, state) + self.assertThat(config_conf_mo_mock, MockCalledOnceWith(power_dn, ANY)) + power_change = config_conf_mo_mock.call_args[0][1][0] + self.assertEqual(power_change.tag, 'lsPower') + self.assertEqual({'state': state, 'dn': power_dn}, power_change.attrib) + + +class TestLoggedIn(MAASTestCase): + """Tests for ``logged_in``.""" + + def test_logged_in(self): + mock = self.patch(ucsm, 'UCSM_XML_API') + url = 'url' + username = 'username' + password = 'password' + mock.return_value = Mock() + + with logged_in(url, username, password) as api: + self.assertEqual(mock.return_value, api) + self.assertThat(api.login, MockCalledOnceWith()) + + self.assertThat(mock.return_value.logout, MockCalledOnceWith()) + + +class TestValidGetPowerCommand(MAASTestCase): + scenarios = [ + ('Power On', dict( + power_mode='on', current_state='down', command='admin-up')), + ('Power On', dict( + power_mode='on', current_state='up', command='cycle-immediate')), + ('Power Off', dict( + power_mode='off', current_state='up', command='admin-down')), + ] + + def test_get_power_command(self): + command = get_power_command(self.power_mode, self.current_state) + self.assertEqual(self.command, command) + + +class TestInvalidGetPowerCommand(MAASTestCase): + + def test_get_power_command_raises_assertion_error_on_bad_power_mode(self): + bad_power_mode = factory.make_name('unlikely') + error = self.assertRaises(AssertionError, get_power_command, + bad_power_mode, None) + self.assertIn(bad_power_mode, error.args[0]) + + +class TestPowerControlUCSM(MAASTestCase): + """Tests for ``power_control_ucsm``.""" + + def test_power_control_ucsm(self): + uuid = factory.getRandomUUID() + api = Mock() + self.patch(ucsm, 'UCSM_XML_API').return_value = api + get_servers_mock = self.patch(ucsm, 'get_servers') + server = make_server() + state = 'admin-down' + power_control = Element('lsPower', {'state': state}) + get_servers_mock.return_value = [server] + get_server_power_control_mock = self.patch(ucsm, + 'get_server_power_control') + get_server_power_control_mock.return_value = power_control + set_server_power_control_mock = self.patch(ucsm, + 'set_server_power_control') + power_control_ucsm('url', 'username', 'password', uuid, + 'off') + self.assertThat(get_servers_mock, MockCalledOnceWith(api, uuid)) + self.assertThat(set_server_power_control_mock, + MockCalledOnceWith(api, power_control, state)) + + +class TestProbeAndEnlistUCSM(MAASTestCase): + """Tests for ``probe_and_enlist_ucsm``.""" + + def test_probe_and_enlist(self): + url = 'url' + username = 'username' + password = 'password' + api = Mock() + self.patch(ucsm, 'UCSM_XML_API').return_value = api + server_element = {'uuid': 'uuid'} + server = (server_element, ['mac'],) + probe_servers_mock = self.patch(ucsm, 'probe_servers') + probe_servers_mock.return_value = [server] + set_lan_boot_default_mock = self.patch(ucsm, 'set_lan_boot_default') + create_node_mock = self.patch(utils, 'create_node') + probe_and_enlist_ucsm(url, username, password) + self.assertThat(set_lan_boot_default_mock, + MockCalledOnceWith(api, server_element)) + self.assertThat(probe_servers_mock, MockCalledOnceWith(api)) + params = { + 'power_address': url, + 'power_user': username, + 'power_pass': password, + 'uuid': server[0]['uuid'] + } + self.assertThat(create_node_mock, + MockCalledOnceWith(server[1], 'amd64', 'ucsm', params)) + + +class TestGetServiceProfile(MAASTestCase): + """Tests for ``get_service_profile.``""" + + def test_get_service_profile(self): + test_dn = make_dn() + server = Element('computeBlade', {'assignedToDn': test_dn}) + api = make_api() + mock = self.patch(api, 'config_resolve_dn') + mock.return_value = make_fake_result('configResolveDn', 'lsServer', + 'outConfig') + service_profile = get_service_profile(api, server) + self.assertThat(mock, MockCalledOnceWith(test_dn)) + self.assertEqual(mock.return_value[0], service_profile) + + +def make_boot_order_scenarios(size): + minimum = random.randint(0, 500) + ordinals = xrange(minimum, minimum + size) + + elements = [ + Element('Entry%d' % i, {'order': '%d' % i}) + for i in ordinals + ] + + orders = permutations(elements) + orders = [{'order': order} for order in orders] + + scenarios = [('%d' % i, order) for i, order in enumerate(orders)] + return scenarios, minimum + + +class TestGetFirstBooter(MAASTestCase): + """Tests for ``get_first_booter.``""" + + scenarios, minimum = make_boot_order_scenarios(3) + + def test_first_booter(self): + root = Element('outConfigs') + root.extend(self.order) + picked = get_first_booter(root) + self.assertEqual(picked.tag, 'Entry%d' % self.minimum) + + +class TestsForStripRoKeys(MAASTestCase): + """Tests for ``strip_ro_keys.``""" + + def test_strip_ro_keys(self): + attributes = {key: 'DC' for key in RO_KEYS} + + elements = [ + Element('Element%d' % i, attributes) + for i in xrange(random.randint(0, 10)) + ] + + strip_ro_keys(elements) + + for key in RO_KEYS: + values = [element.get(key) for element in elements] + for value in values: + self.assertIsNone(value) + + +class TestMakePolicyChange(MAASTestCase): + """Tests for ``make_policy_change``.""" + + def test_lan_already_top_priority(self): + boot_profile_response = make_fake_result('configResolveChildren', + 'lsbootLan') + mock = self.patch(ucsm, 'get_first_booter') + mock.return_value = boot_profile_response[0] + change = make_policy_change(boot_profile_response) + self.assertIsNone(change) + self.assertThat(mock, MockCalledOnceWith(boot_profile_response)) + + def test_change_lan_to_top_priority(self): + boot_profile_response = Element('outConfigs') + lan_boot = Element('lsbootLan', {'order': 'second'}) + storage_boot = Element('lsbootStorage', {'order': 'first'}) + boot_profile_response.extend([lan_boot, storage_boot]) + self.patch(ucsm, 'get_first_booter').return_value = storage_boot + self.patch(ucsm, 'strip_ro_keys') + change = make_policy_change(boot_profile_response) + lan_boot_order = change.xpath('//lsbootPolicy/lsbootLan/@order') + storage_boot_order = \ + change.xpath('//lsbootPolicy/lsbootStorage/@order') + self.assertEqual(['first'], lan_boot_order) + self.assertEqual(['second'], storage_boot_order) + + +class TestSetLanBootDefault(MAASTestCase): + """Tets for ``set_lan_boot_default.``""" + + def test_no_change(self): + api = make_api() + server = make_server() + self.patch(ucsm, 'get_service_profile') + self.patch(api, 'config_resolve_children') + self.patch(ucsm, 'make_policy_change').return_value = None + config_conf_mo = self.patch(api, 'config_conf_mo') + set_lan_boot_default(api, server) + self.assertThat(config_conf_mo, MockNotCalled()) + + def test_with_change(self): + api = make_api() + server = make_server() + test_dn = make_dn() + test_change = 'change' + service_profile = Element('test', {'operBootPolicyName': test_dn}) + self.patch(ucsm, 'get_service_profile').return_value = service_profile + self.patch(api, 'config_resolve_children') + self.patch(ucsm, 'make_policy_change').return_value = test_change + config_conf_mo = self.patch(api, 'config_conf_mo') + set_lan_boot_default(api, server) + self.assertThat(config_conf_mo, + MockCalledOnceWith(test_dn, [test_change])) diff -Nru maas-1.5+bzr2252/src/provisioningserver/custom_hardware/tests/test_virsh.py maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/tests/test_virsh.py --- maas-1.5+bzr2252/src/provisioningserver/custom_hardware/tests/test_virsh.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/tests/test_virsh.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,351 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Tests for `provisioningserver.custom_hardware.virsh`. +""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [] + +import random +from textwrap import dedent + +from maastesting.factory import factory +from maastesting.matchers import ( + MockCalledOnceWith, + MockCallsMatch, + ) +from maastesting.testcase import MAASTestCase +from mock import call +from provisioningserver.custom_hardware import ( + utils, + virsh, + ) + + +SAMPLE_IFLIST = dedent(""" + Interface Type Source Model MAC + ------------------------------------------------------- + - bridge br0 e1000 %s + - bridge br1 e1000 %s + """) + +SAMPLE_DUMPXML = dedent(""" + + test + 4096576 + 4096576 + 1 + + hvm + + + + """) + + +class TestVirshSSH(MAASTestCase): + """Tests for `VirshSSH`.""" + + def configure_virshssh_pexpect(self, inputs=None): + """Configures the VirshSSH class to use 'cat' process + for testing instead of the actual virsh.""" + conn = virsh.VirshSSH(timeout=1) + self.addCleanup(conn.close) + self.patch(conn, '_execute') + conn._spawn('cat') + if inputs is not None: + for line in inputs: + conn.sendline(line) + return conn + + def configure_virshssh(self, output): + self.patch(virsh.VirshSSH, 'run').return_value = output + return virsh.VirshSSH() + + def test_login_prompt(self): + virsh_outputs = [ + 'virsh # ' + ] + conn = self.configure_virshssh_pexpect(virsh_outputs) + self.assertTrue(conn.login(poweraddr=None)) + + def test_login_with_sshkey(self): + virsh_outputs = [ + "The authenticity of host '127.0.0.1' can't be established.", + "ECDSA key fingerprint is " + "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff.", + "Are you sure you want to continue connecting (yes/no)? ", + ] + conn = self.configure_virshssh_pexpect(virsh_outputs) + mock_sendline = self.patch(conn, 'sendline') + conn.login(poweraddr=None) + self.assertThat(mock_sendline, MockCalledOnceWith('yes')) + + def test_login_with_password(self): + virsh_outputs = [ + "ubuntu@%s's password: " % factory.getRandomIPAddress(), + ] + conn = self.configure_virshssh_pexpect(virsh_outputs) + fake_password = factory.make_name('password') + mock_sendline = self.patch(conn, 'sendline') + conn.login(poweraddr=None, password=fake_password) + self.assertThat(mock_sendline, MockCalledOnceWith(fake_password)) + + def test_login_missing_password(self): + virsh_outputs = [ + "ubuntu@%s's password: " % factory.getRandomIPAddress(), + ] + conn = self.configure_virshssh_pexpect(virsh_outputs) + mock_close = self.patch(conn, 'close') + self.assertFalse(conn.login(poweraddr=None, password=None)) + mock_close.assert_called() + + def test_login_invalid(self): + virsh_outputs = [ + factory.getRandomString(), + ] + conn = self.configure_virshssh_pexpect(virsh_outputs) + mock_close = self.patch(conn, 'close') + self.assertFalse(conn.login(poweraddr=None)) + mock_close.assert_called() + + def test_logout(self): + conn = self.configure_virshssh_pexpect() + mock_sendline = self.patch(conn, 'sendline') + mock_close = self.patch(conn, 'close') + conn.logout() + self.assertThat(mock_sendline, MockCalledOnceWith('quit')) + mock_close.assert_called() + + def test_prompt(self): + virsh_outputs = [ + 'virsh # ' + ] + conn = self.configure_virshssh_pexpect(virsh_outputs) + self.assertTrue(conn.prompt()) + + def test_invalid_prompt(self): + virsh_outputs = [ + factory.getRandomString() + ] + conn = self.configure_virshssh_pexpect(virsh_outputs) + self.assertFalse(conn.prompt()) + + def test_run(self): + cmd = ['list', '--all', '--name'] + expected = ' '.join(cmd) + names = [factory.make_name('machine') for _ in range(3)] + conn = self.configure_virshssh_pexpect() + conn.before = '\n'.join([expected] + names) + mock_sendline = self.patch(conn, 'sendline') + mock_prompt = self.patch(conn, 'prompt') + output = conn.run(cmd) + self.assertThat(mock_sendline, MockCalledOnceWith(expected)) + mock_prompt.assert_called() + self.assertEqual('\n'.join(names), output) + + def test_list(self): + names = [factory.make_name('machine') for _ in range(3)] + conn = self.configure_virshssh('\n'.join(names)) + expected = conn.list() + self.assertItemsEqual(names, expected) + + def test_get_state(self): + state = factory.make_name('state') + conn = self.configure_virshssh(state) + expected = conn.get_state('') + self.assertEqual(state, expected) + + def test_get_state_error(self): + conn = self.configure_virshssh('error') + expected = conn.get_state('') + self.assertEqual(None, expected) + + def test_mac_addresses_returns_list(self): + macs = [factory.getRandomMACAddress() for _ in range(2)] + output = SAMPLE_IFLIST % (macs[0], macs[1]) + conn = self.configure_virshssh(output) + expected = conn.get_mac_addresses('') + for i in range(2): + self.assertEqual(macs[i], expected[i]) + + def test_get_arch_returns_valid(self): + arch = factory.make_name('arch') + output = SAMPLE_DUMPXML % arch + conn = self.configure_virshssh(output) + expected = conn.get_arch('') + self.assertEqual(arch, expected) + + def test_get_arch_returns_valid_fixed(self): + arch = random.choice(virsh.ARCH_FIX.keys()) + fixed_arch = virsh.ARCH_FIX[arch] + output = SAMPLE_DUMPXML % arch + conn = self.configure_virshssh(output) + expected = conn.get_arch('') + self.assertEqual(fixed_arch, expected) + + +class TestVirsh(MAASTestCase): + """Tests for `probe_virsh_and_enlist`.""" + + def test_probe_and_enlist(self): + # Patch VirshSSH list so that some machines are returned + # with some fake architectures. + machines = [factory.make_name('machine') for _ in range(3)] + self.patch(virsh.VirshSSH, 'list').return_value = machines + fake_arch = factory.make_name('arch') + mock_arch = self.patch(virsh.VirshSSH, 'get_arch') + mock_arch.return_value = fake_arch + + # Patch get_state so that one of the machines is on, so we + # can check that it will be forced off. + fake_states = [ + virsh.VirshVMState.ON, + virsh.VirshVMState.OFF, + virsh.VirshVMState.OFF + ] + mock_state = self.patch(virsh.VirshSSH, 'get_state') + mock_state.side_effect = fake_states + + # Setup the power parameters that we should expect to be + # the output of the probe_and_enlist + fake_password = factory.getRandomString() + poweraddr = factory.make_name('poweraddr') + called_params = [] + fake_macs = [] + for machine in machines: + macs = [factory.getRandomMACAddress() for _ in range(3)] + fake_macs.append(macs) + called_params.append({ + 'power_address': poweraddr, + 'power_id': machine, + 'power_pass': fake_password, + }) + + # Patch the get_mac_addresses so we get a known list of + # mac addresses for each machine. + mock_macs = self.patch(virsh.VirshSSH, 'get_mac_addresses') + mock_macs.side_effect = fake_macs + + # Patch the poweroff and create as we really don't want these + # actions to occur, but want to also check that they are called. + mock_poweroff = self.patch(virsh.VirshSSH, 'poweroff') + mock_create = self.patch(utils, 'create_node') + + # Patch login and logout so that we don't really contact + # a server at the fake poweraddr + mock_login = self.patch(virsh.VirshSSH, 'login') + mock_login.return_value = True + mock_logout = self.patch(virsh.VirshSSH, 'logout') + + # Perform the probe and enlist + virsh.probe_virsh_and_enlist(poweraddr, password=fake_password) + + # Check that login was called with the provided poweraddr and + # password. + self.assertThat( + mock_login, MockCalledOnceWith(poweraddr, fake_password)) + + # The first machine should have poweroff called on it, as it + # was initial in the on state. + self.assertThat( + mock_poweroff, MockCalledOnceWith(machines[0])) + + # Check that the create command had the correct parameters for + # each machine. + self.assertThat( + mock_create, MockCallsMatch( + call(fake_macs[0], fake_arch, 'virsh', called_params[0]), + call(fake_macs[1], fake_arch, 'virsh', called_params[1]), + call(fake_macs[2], fake_arch, 'virsh', called_params[2]))) + mock_logout.assert_called() + + def test_probe_and_enlist_login_failure(self): + mock_login = self.patch(virsh.VirshSSH, 'login') + mock_login.return_value = False + self.assertRaises( + virsh.VirshError, virsh.probe_virsh_and_enlist, + factory.make_name('poweraddr'), password=factory.getRandomString()) + + +class TestVirshPowerControl(MAASTestCase): + """Tests for `power_control_virsh`.""" + + def test_power_control_login_failure(self): + mock_login = self.patch(virsh.VirshSSH, 'login') + mock_login.return_value = False + self.assertRaises( + virsh.VirshError, virsh.power_control_virsh, + factory.make_name('poweraddr'), factory.make_name('machine'), + 'on', password=factory.getRandomString()) + + def test_power_control_on(self): + mock_login = self.patch(virsh.VirshSSH, 'login') + mock_login.return_value = True + mock_state = self.patch(virsh.VirshSSH, 'get_state') + mock_state.return_value = virsh.VirshVMState.OFF + mock_poweron = self.patch(virsh.VirshSSH, 'poweron') + + poweraddr = factory.make_name('poweraddr') + machine = factory.make_name('machine') + virsh.power_control_virsh(poweraddr, machine, 'on') + + self.assertThat( + mock_login, MockCalledOnceWith(poweraddr, None)) + self.assertThat( + mock_state, MockCalledOnceWith(machine)) + self.assertThat( + mock_poweron, MockCalledOnceWith(machine)) + + def test_power_control_off(self): + mock_login = self.patch(virsh.VirshSSH, 'login') + mock_login.return_value = True + mock_state = self.patch(virsh.VirshSSH, 'get_state') + mock_state.return_value = virsh.VirshVMState.ON + mock_poweroff = self.patch(virsh.VirshSSH, 'poweroff') + + poweraddr = factory.make_name('poweraddr') + machine = factory.make_name('machine') + virsh.power_control_virsh(poweraddr, machine, 'off') + + self.assertThat( + mock_login, MockCalledOnceWith(poweraddr, None)) + self.assertThat( + mock_state, MockCalledOnceWith(machine)) + self.assertThat( + mock_poweroff, MockCalledOnceWith(machine)) + + def test_power_control_bad_domain(self): + mock_login = self.patch(virsh.VirshSSH, 'login') + mock_login.return_value = True + mock_state = self.patch(virsh.VirshSSH, 'get_state') + mock_state.return_value = None + + poweraddr = factory.make_name('poweraddr') + machine = factory.make_name('machine') + self.assertRaises( + virsh.VirshError, virsh.power_control_virsh, + poweraddr, machine, 'on') + + def test_power_control_power_failure(self): + mock_login = self.patch(virsh.VirshSSH, 'login') + mock_login.return_value = True + mock_state = self.patch(virsh.VirshSSH, 'get_state') + mock_state.return_value = virsh.VirshVMState.ON + mock_poweroff = self.patch(virsh.VirshSSH, 'poweroff') + mock_poweroff.return_value = False + + poweraddr = factory.make_name('poweraddr') + machine = factory.make_name('machine') + self.assertRaises( + virsh.VirshError, virsh.power_control_virsh, + poweraddr, machine, 'off') diff -Nru maas-1.5+bzr2252/src/provisioningserver/custom_hardware/ucsm.py maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/ucsm.py --- maas-1.5+bzr2252/src/provisioningserver/custom_hardware/ucsm.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/ucsm.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,435 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Support for managing nodes via Cisco UCS Manager's HTTP-XML API. + +It's useful to have a cursory understanding of how UCS Manager XML API +works. Cisco has a proprietary document that describes all of this in +more detail, and I would suggest you get a copy of that if you want more +information than is provided here. + +The Cisco DevNet website for UCS Manager has a link to the document, +which is behind a login wall, and links to example UCS queries: + +https://developer.cisco.com/web/unifiedcomputing/home + +UCS Manager is a tool for managing servers. It provides an XML API for +external applications to use to interact with UCS Manager to manage +servers. The API is available via HTTP, and requests and responses are +made of XML strings. MAAS's code for interacting with a UCS Manager is +concerned with building these requests, sending them to UCS Manager, and +processing the responses. + +UCS Manager stores information in a hierarchical structure known as the +management information tree. This structure is exposed via the XML API, +where we can manipulate objects in the tree by finding them, reading +them, and writing them. + +Some definitions for terms that are used in this code: + +Boot Policy - Controls the boot order for a server. Each service profile +is associated with a boot policy. + +Distinguished Name (DN) - Each object in UCS has a unique DN, which +describes its position in the tree. This is like a fully qualified path, +and provides a way for objects to reference other objects at other +places in the tree, or for API users to look up specific objects in the +tree. + +Class - Classes define the properties and states of objects. An object's +class is given in its tag name. + +Managed Object (MO) - An object in the management information tree. +Objects are recursive, and may have children of multiple types. With the +exception of the root object, all objects have parents. In the XML API, +objects are represented as XML elements. + +Method - Actions performed by the API on managed objects. These can +change state, or read the current state, or both. + +Server - A physical server managed by UCS Manager. Servers must be +associated with service profiles in order to be used. + +Service Profile - A set of configuration options for a server. Service +profiles define the server's personality, and can be migrated from +server to server. Service profiles describe boot policy, MAC addresses, +network connectivity, IPMI configuration, and more. MAAS requires +servers to be associated with service profiles. + +UUID - The UUID for a server. MAAS persists the UUID of each UCS managed +server it enlists, and uses it as a key for looking the server up later. +""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +import contextlib +import urllib2 +import urlparse + +from lxml.etree import ( + Element, + tostring, + XML, + ) +import provisioningserver.custom_hardware.utils as utils + + +str = None + +__metaclass__ = type +__all__ = [ + 'power_control_ucsm', + 'probe_and_enlist_ucsm', +] + + +class UCSM_XML_API_Error(Exception): + """Failure talking to a Cisco UCS Manager.""" + + def __init__(self, msg, code): + super(UCSM_XML_API_Error, self).__init__(msg) + self.code = code + + +def make_request_data(name, fields=None, children=None): + """Build a request string for an API method.""" + root = Element(name, fields) + if children is not None: + root.extend(children) + return tostring(root) + + +def parse_response(response_string): + """Parse the response from an API method.""" + doc = XML(response_string) + + error_code = doc.get('errorCode') + if error_code is not None: + raise UCSM_XML_API_Error(doc.get('errorDescr'), error_code) + + return doc + + +class UCSM_XML_API(object): + """Provides access to a Cisco UCS Manager's XML API. Public methods + on this class correspond to UCS Manager XML API methods. + + Each request uses a new connection. The server supports keep-alive, + so this client could be optimized to use it too. + """ + + def __init__(self, url, username, password): + self.url = url + self.api_url = urlparse.urljoin(self.url, 'nuova') + self.username = username + self.password = password + self.cookie = None + + def _send_request(self, request_data): + """Issue a request via HTTP and parse the response.""" + request = urllib2.Request(self.api_url, request_data) + response = urllib2.urlopen(request) + response_text = response.read() + response_doc = parse_response(response_text) + return response_doc + + def _call(self, name, fields=None, children=None): + request_data = make_request_data(name, fields, children) + response = self._send_request(request_data) + return response + + def login(self): + """Login to the API and get a cookie. + + Logging into the API gives a new cookie in response. The cookie + will become inactive after it has been inactive for some amount + of time (10 minutes is the default.) + + UCS Manager allows a limited number of active cookies at any + point in time, so it's important to free the cookie up when + finished by logging out via the ``logout`` method. + """ + fields = {'inName': self.username, 'inPassword': self.password} + response = self._call('aaaLogin', fields) + self.cookie = response.get('outCookie') + + def logout(self): + """Logout from the API and free the cookie.""" + fields = {'inCookie': self.cookie} + self._call('aaaLogout', fields) + self.cookie = None + + def config_resolve_class(self, class_id, filters=None): + """Issue a configResolveClass request. + + This returns all of the objects of class ``class_id`` from the + UCS Manager. + + Filters provide a way of limiting the classes returned according + to their attributes. There are a number of filters available - + Cisco's XML API documentation has a full chapter on filters. + All we care about here is that filters are described with XML + elements. + """ + fields = {'cookie': self.cookie, 'classId': class_id} + + in_filters = Element('inFilter') + if filters: + in_filters.extend(filters) + + return self._call('configResolveClass', fields, [in_filters]) + + def config_resolve_children(self, dn, class_id=None): + """Issue a configResolveChildren request. + + This returns all of the children of the object named by ``dn``, + or if ``class_id`` is not None, all of the children of type + ``class_id``. + """ + fields = {'cookie': self.cookie, 'inDn': dn} + if class_id is not None: + fields['classId'] = class_id + return self._call('configResolveChildren', fields) + + def config_resolve_dn(self, dn): + """Retrieve a single object by name. + + This returns the object named by ``dn``, but not its children. + """ + fields = {'cookie': self.cookie, 'dn': dn} + return self._call('configResolveDn', fields) + + def config_conf_mo(self, dn, config_items): + """Issue a configConfMo request. + + This makes a configuration change on an object (MO). + """ + fields = {'cookie': self.cookie, 'dn': dn} + + in_configs = Element('inConfig') + in_configs.extend(config_items) + + self._call('configConfMo', fields, [in_configs]) + + +def get_servers(api, uuid=None): + """Retrieve a list of servers from the UCS Manager.""" + if uuid: + attrs = {'class': 'computeItem', 'property': 'uuid', 'value': uuid} + filters = [Element('eq', attrs)] + else: + filters = None + + resolved = api.config_resolve_class('computeItem', filters) + return resolved.xpath('//outConfigs/*') + + +def get_children(api, element, class_id): + """Retrieve a list of child elements from the UCS Manager.""" + resolved = api.config_resolve_children(element.get('dn'), class_id) + return resolved.xpath('//outConfigs/%s' % class_id) + + +def get_macs(api, server): + """Retrieve the list of MAC addresses assigned to a server. + + Network interfaces are represented by 'adaptorUnit' objects, and + are stored as children of servers. + """ + adaptors = get_children(api, server, 'adaptorUnit') + + macs = [] + for adaptor in adaptors: + host_eth_ifs = get_children(api, adaptor, 'adaptorHostEthIf') + macs.extend([h.get('mac') for h in host_eth_ifs]) + + return macs + + +def probe_servers(api): + """Retrieve the UUID and MAC addresses for servers from the UCS Manager.""" + servers = get_servers(api) + server_list = [(s, get_macs(api, s)) for s in servers] + return server_list + + +def get_server_power_control(api, server): + """Retrieve the power control object for a server.""" + service_profile_dn = server.get('assignedToDn') + resolved = api.config_resolve_children(service_profile_dn, 'lsPower') + power_controls = resolved.xpath('//outConfigs/lsPower') + return power_controls[0] + + +def set_server_power_control(api, power_control, command): + """Issue a power command to a server's power control.""" + attrs = {'state': command, 'dn': power_control.get('dn')} + power_change = Element('lsPower', attrs) + api.config_conf_mo(power_control.get('dn'), [power_change]) + + +def get_service_profile(api, server): + """Get the server's assigned service profile.""" + service_profile_dn = server.get('assignedToDn') + result = api.config_resolve_dn(service_profile_dn) + service_profile = result.xpath('//outConfig/lsServer')[0] + return service_profile + + +def get_first_booter(boot_profile_response): + """Find the device currently set to boot by default.""" + ordinals = boot_profile_response.xpath('//outConfigs/*/@order') + top_boot_order = min(ordinals) + first_query = '//outConfigs/*[@order=%s]' % top_boot_order + current_first = boot_profile_response.xpath(first_query)[0] + return current_first + + +RO_KEYS = ['access', 'type'] + + +def strip_ro_keys(elements): + """Remove read-only keys from configuration elements. + + These are keys for attributes that aren't allowed to be changed via + configConfMo request. They are included in MO's that we read from the + API; stripping these attributes lets us reuse the elements for those + MO's rather than building new ones from scratch. + """ + for ro_key in RO_KEYS: + for element in elements: + del(element.attrib[ro_key]) + + +def make_policy_change(boot_profile_response): + """Build the policy change tree required to make LAN boot first + priority. + + The original top priority will be swapped with LAN boot's original + priority. + """ + current_first = get_first_booter(boot_profile_response) + lan_boot = boot_profile_response.xpath('//outConfigs/lsbootLan')[0] + + if current_first == lan_boot: + return + + top_boot_order = current_first.get('order') + current_first.set('order', lan_boot.get('order')) + lan_boot.set('order', top_boot_order) + + elements = [current_first, lan_boot] + strip_ro_keys(elements) + policy_change = Element('lsbootPolicy') + policy_change.extend(elements) + return policy_change + + +def set_lan_boot_default(api, server): + """Set a server to boot via LAN by default. + + If LAN boot is already the top priority, no change will + be made. + + This command changes the server's boot profile, which will affect + any other servers also using that boot profile. This is ok, because + probe and enlist enlists all the servers in the chassis. + """ + service_profile = get_service_profile(api, server) + boot_profile_dn = service_profile.get('operBootPolicyName') + response = api.config_resolve_children(boot_profile_dn) + policy_change = make_policy_change(response) + if policy_change is None: + return + api.config_conf_mo(boot_profile_dn, [policy_change]) + + +@contextlib.contextmanager +def logged_in(url, username, password): + """Context manager that ensures the logout from the API occurs.""" + api = UCSM_XML_API(url, username, password) + api.login() + try: + yield api + finally: + api.logout() + + +def get_power_command(maas_power_mode, current_state): + """Translate a MAAS on/off state into a UCSM power command. + + If the node is up already and receives a request to power on, power + cycle the node. + """ + if maas_power_mode == 'on': + if current_state == 'up': + return 'cycle-immediate' + return 'admin-up' + elif maas_power_mode == 'off': + return 'admin-down' + else: + message = 'Unexpected maas power mode: %s' % (maas_power_mode) + raise AssertionError(message) + + +def power_control_ucsm(url, username, password, uuid, maas_power_mode): + """Handle calls from the power template for nodes with a power type + of 'ucsm'. + """ + with logged_in(url, username, password) as api: + # UUIDs are unique per server, so we get either one or zero + # servers for a given UUID. + [server] = get_servers(api, uuid) + power_control = get_server_power_control(api, server) + command = get_power_command(maas_power_mode, + power_control.get('state')) + set_server_power_control(api, power_control, command) + + +def probe_and_enlist_ucsm(url, username, password): + """Probe a UCS Manager and enlist all its servers. + + Here's what happens here: 1. Get a list of servers from the UCS + Manager, along with their MAC addresses. + + 2. Configure each server to boot from LAN first. + + 3. Add each server to MAAS as a new node, with a power control + method of 'ucsm'. The URL and credentials supplied are persisted + with each node so MAAS knows how to access UCSM to manage the node + in the future. + + This code expects each server in the system to have already been + associated with a service profile. The servers must have networking + configured, and their boot profiles must include a boot from LAN + option. During enlistment, the boot profile for each service profile + used by a server will be modified to move LAN boot to the highest + priority boot option. + + Also, if any node fails to enlist, this enlistment process will + stop and won't attempt to enlist any additional nodes. If a node is + already known to MAAS, it will fail to enlist, so all nodes must be + added at once. + + There is also room for optimization during enlistment. While our + client deals with a single server at a time, the API is capable + of reading/writing the settings of multiple servers in the same + request. + """ + with logged_in(url, username, password) as api: + servers = probe_servers(api) + for server, _ in servers: + set_lan_boot_default(api, server) + + for server, macs in servers: + params = { + 'power_address': url, + 'power_user': username, + 'power_pass': password, + 'uuid': server.get('uuid'), + } + utils.create_node(macs, 'amd64', 'ucsm', params) diff -Nru maas-1.5+bzr2252/src/provisioningserver/custom_hardware/utils.py maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/utils.py --- maas-1.5+bzr2252/src/provisioningserver/custom_hardware/utils.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/utils.py 2014-09-03 14:18:31.000000000 +0000 @@ -42,3 +42,7 @@ 'autodetect_nodegroup': 'true' } return client.post('/api/1.0/nodes/', 'new', **data) + + +def escape_string(data): + return repr(data).decode("ascii") diff -Nru maas-1.5+bzr2252/src/provisioningserver/custom_hardware/virsh.py maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/virsh.py --- maas-1.5+bzr2252/src/provisioningserver/custom_hardware/virsh.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/custom_hardware/virsh.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,223 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [ + 'probe_virsh_and_enlist', + ] + +from lxml import etree +import pexpect +import provisioningserver.custom_hardware.utils as utils + + +XPATH_ARCH = "/domain/os/type/@arch" + +# Virsh stores the architecture with a different +# label then MAAS. This maps virsh architecture to +# MAAS architecture. +ARCH_FIX = { + 'x86_64': 'amd64', + 'ppc64': 'ppc64el', + } + + +class VirshVMState: + OFF = "shut off" + ON = "running" + + +class VirshError(Exception): + """Failure communicating to virsh. """ + + +class VirshSSH(pexpect.spawn): + + PROMPT = r"virsh \#" + PROMPT_SSHKEY = "(?i)are you sure you want to continue connecting" + PROMPT_PASSWORD = "(?i)(?:password)|(?:passphrase for key)" + PROMPT_DENIED = "(?i)permission denied" + PROMPT_CLOSED = "(?i)connection closed by remote host" + + PROMPTS = [ + PROMPT_SSHKEY, + PROMPT_PASSWORD, + PROMPT, + PROMPT_DENIED, + PROMPT_CLOSED, + pexpect.TIMEOUT, + pexpect.EOF, + ] + + I_PROMPT = PROMPTS.index(PROMPT) + I_PROMPT_SSHKEY = PROMPTS.index(PROMPT_SSHKEY) + I_PROMPT_PASSWORD = PROMPTS.index(PROMPT_PASSWORD) + + def __init__(self, timeout=30, maxread=2000): + super(VirshSSH, self).__init__( + None, timeout=timeout, maxread=maxread) + self.name = '' + + def _execute(self, poweraddr): + """Spawns the pexpect command.""" + cmd = 'virsh --connect %s' % poweraddr + self._spawn(cmd) + + def login(self, poweraddr, password=None): + """Starts connection to virsh.""" + self._execute(poweraddr) + i = self.expect(self.PROMPTS, timeout=10) + if i == self.I_PROMPT_SSHKEY: + # New certificate, lets always accept but if + # it changes it will fail to login. + self.sendline("yes") + i = self.expect(self.PROMPTS) + elif i == self.I_PROMPT_PASSWORD: + # Requesting password, give it if available. + if password is None: + self.close() + return False + self.sendline(password) + i = self.expect(self.PROMPTS) + + if i != self.I_PROMPT: + # Something bad happened, either disconnect, + # timeout, wrong password. + self.close() + return False + return True + + def logout(self): + """Quits the virsh session.""" + self.sendline("quit") + self.close() + + def prompt(self, timeout=None): + """Waits for virsh prompt.""" + if timeout is None: + timeout = self.timeout + i = self.expect([self.PROMPT, pexpect.TIMEOUT], timeout=timeout) + if i == 1: + return False + return True + + def run(self, args): + cmd = ' '.join(args) + self.sendline(cmd) + self.prompt() + result = self.before.splitlines() + return '\n'.join(result[1:]) + + def list(self): + """Lists all virtual machines by name.""" + machines = self.run(['list', '--all', '--name']) + return machines.strip().splitlines() + + def get_state(self, machine): + """Gets the virtual machine state.""" + state = self.run(['domstate', machine]) + state = state.strip() + if 'error' in state: + return None + return state + + def get_mac_addresses(self, machine): + """Gets list of mac addressess assigned to the virtual machine.""" + output = self.run(['domiflist', machine]).strip() + if 'error' in output: + return None + output = output.splitlines()[2:] + return [line.split()[4] for line in output] + + def get_arch(self, machine): + """Gets the virtual machine architecture.""" + output = self.run(['dumpxml', machine]).strip() + if 'error' in output: + return None + + doc = etree.XML(output) + evaluator = etree.XPathEvaluator(doc) + arch = evaluator(XPATH_ARCH)[0] + + # Fix architectures that need to be referenced by a different + # name, that MAAS understands. + return ARCH_FIX.get(arch, arch) + + def poweron(self, machine): + """Poweron a virtual machine.""" + output = self.run(['start', machine]).strip() + if 'error' in output: + return False + return True + + def poweroff(self, machine): + """Poweroff a virtual machine.""" + output = self.run(['destroy', machine]).strip() + if 'error' in output: + return False + return True + + +def probe_virsh_and_enlist(poweraddr, password=None): + """Extracts all of the virtual machines from virsh and enlists them + into MAAS. + + :param poweraddr: virsh connection string + """ + conn = VirshSSH() + if not conn.login(poweraddr, password): + raise VirshError('Failed to login to virsh console.') + + for machine in conn.list(): + arch = conn.get_arch(machine) + state = conn.get_state(machine) + macs = conn.get_mac_addresses(machine) + + # Force the machine off, as MAAS will control the machine + # and it needs to be in a known state of off. + if state == VirshVMState.ON: + conn.poweroff(machine) + + params = { + 'power_address': poweraddr, + 'power_id': machine, + } + if password is not None: + params['power_pass'] = password + utils.create_node(macs, arch, 'virsh', params) + + conn.logout() + + +def power_control_virsh(poweraddr, machine, power_change, password=None): + """Powers controls a virtual machine using virsh.""" + + # Force password to None if blank, as the power control + # script will send a blank password if one is not set. + if password == '': + password = None + + conn = VirshSSH() + if not conn.login(poweraddr, password): + raise VirshError('Failed to login to virsh console.') + + state = conn.get_state(machine) + if state is None: + raise VirshError('Failed to get domain: %s' % machine) + + if state == VirshVMState.OFF: + if power_change == 'on': + if conn.poweron(machine) is False: + raise VirshError('Failed to power on domain: %s' % machine) + elif state == VirshVMState.ON: + if power_change == 'off': + if conn.poweroff(machine) is False: + raise VirshError('Failed to power off domain: %s' % machine) diff -Nru maas-1.5+bzr2252/src/provisioningserver/dhcp/config.py maas-1.5.4+bzr2294/src/provisioningserver/dhcp/config.py --- maas-1.5+bzr2252/src/provisioningserver/dhcp/config.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/dhcp/config.py 2014-09-03 14:18:31.000000000 +0000 @@ -33,16 +33,22 @@ # Used to generate the conditional bootloader behaviour CONDITIONAL_BOOTLOADER = """ -{behaviour} option arch = {arch_octet} {{ - filename \"{bootloader}\"; - }} +{{behaviour}} option arch = {{arch_octet}} { + filename \"{{bootloader}}\"; + {{if path_prefix}} + option path-prefix \"{{path_prefix}}\"; + {{endif}} + } """ # Used to generate the PXEBootLoader special case PXE_BOOTLOADER = """ -else {{ - filename \"{bootloader}\"; - }} +else { + filename \"{{bootloader}}\"; + {{if path_prefix}} + option path-prefix \"{{path_prefix}}\"; + {{endif}} + } """ @@ -55,9 +61,13 @@ behaviour = chain(["if"], repeat("elsif")) for name, method in BootMethodRegistry: if name != "pxe": - output += CONDITIONAL_BOOTLOADER.format( - behaviour=next(behaviour), arch_octet=method.arch_octet, - bootloader=method.bootloader_path).strip() + ' ' + output += tempita.sub( + CONDITIONAL_BOOTLOADER, + behaviour=next(behaviour), + arch_octet=method.arch_octet, + bootloader=method.bootloader_path, + path_prefix=method.path_prefix, + ).strip() + ' ' # The PXEBootMethod is used in an else statement for the generated # dhcpd config. This ensures that a booting node that does not @@ -65,8 +75,11 @@ # pxelinux can still boot. pxe_method = BootMethodRegistry.get_item('pxe') if pxe_method is not None: - output += PXE_BOOTLOADER.format( - bootloader=pxe_method.bootloader_path).strip() + output += tempita.sub( + PXE_BOOTLOADER, + bootloader=pxe_method.bootloader_path, + path_prefix=pxe_method.path_prefix, + ).strip() return output.strip() diff -Nru maas-1.5+bzr2252/src/provisioningserver/dhcp/leases_parser_fast.py maas-1.5.4+bzr2294/src/provisioningserver/dhcp/leases_parser_fast.py --- maas-1.5+bzr2252/src/provisioningserver/dhcp/leases_parser_fast.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/dhcp/leases_parser_fast.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,87 @@ +# Copyright 2013 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""A speedier version of `leases_parser`. + +This extracts the relevant stanzas from a leases file, keeping only the +most recent "host" and "lease" entries, then uses the existing and +properly defined but slow parser to parse them. This massively speeds up +parsing a leases file that contains a modest number of unique host and +lease entries, but has become very large because of churn. +""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [ + 'parse_leases', + ] + +from collections import defaultdict +from datetime import datetime +from itertools import chain +import re + +from provisioningserver.dhcp.leases_parser import ( + get_host_mac, + has_expired, + is_host, + is_lease, + lease_parser, + ) + + +re_entry = re.compile( + r''' + ^\s* # Ignore leading whitespace on each line. + (host|lease) # Look only for host or lease stanzas. + \s+ # Mandatory whitespace. + ([0-9a-fA-F.:]+) # Capture the IP/MAC address for this stanza. + \s*{ # Optional whitespace then an opening brace. + ''', + re.MULTILINE | re.DOTALL | re.VERBOSE) + + +def find_lease_starts(leases_contents): + results = defaultdict(dict) + for match in re_entry.finditer(leases_contents): + stanza, address = match.groups() + results[stanza][address] = match.start() + return chain.from_iterable( + mapping.itervalues() for mapping in results.itervalues()) + + +def extract_leases(leases_contents): + starts = find_lease_starts(leases_contents) + for start in sorted(starts): + record = lease_parser.scanString(leases_contents[start:]) + try: + token, _, _ = next(record) + except StopIteration: + pass + else: + yield token + + +def parse_leases(leases_contents): + results = {} + now = datetime.utcnow() + for entry in extract_leases(leases_contents): + if is_lease(entry): + if not has_expired(entry, now): + results[entry.ip] = entry.hardware.mac + elif is_host(entry): + mac = get_host_mac(entry) + if mac is None: + # TODO: Test this. + if entry.ip in results: + del results[entry.ip] + else: + results[entry.ip] = mac + return results diff -Nru maas-1.5+bzr2252/src/provisioningserver/dhcp/leases.py maas-1.5.4+bzr2294/src/provisioningserver/dhcp/leases.py --- maas-1.5+bzr2252/src/provisioningserver/dhcp/leases.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/dhcp/leases.py 2014-09-03 14:18:31.000000000 +0000 @@ -54,7 +54,7 @@ get_recorded_nodegroup_uuid, ) from provisioningserver.cluster_config import get_maas_url -from provisioningserver.dhcp.leases_parser import parse_leases +from provisioningserver.dhcp.leases_parser_fast import parse_leases logger = getLogger(__name__) diff -Nru maas-1.5+bzr2252/src/provisioningserver/dhcp/tests/test_leases_parser.py maas-1.5.4+bzr2294/src/provisioningserver/dhcp/tests/test_leases_parser.py --- maas-1.5+bzr2252/src/provisioningserver/dhcp/tests/test_leases_parser.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/dhcp/tests/test_leases_parser.py 2014-09-03 14:18:31.000000000 +0000 @@ -20,6 +20,10 @@ from maastesting.factory import factory from maastesting.testcase import MAASTestCase +from provisioningserver.dhcp import ( + leases_parser, + leases_parser_fast, + ) from provisioningserver.dhcp.leases_parser import ( combine_entries, gather_hosts, @@ -30,197 +34,51 @@ is_host, is_lease, lease_parser, - parse_leases, ) -class TestLeasesParser(MAASTestCase): - - def fake_parsed_lease(self, ip=None, mac=None, ends=None, - entry_type='lease'): - """Fake a lease as produced by the parser.""" - if ip is None: - ip = factory.getRandomIPAddress() - if mac is None: - mac = factory.getRandomMACAddress() - Hardware = namedtuple('Hardware', ['mac']) - Lease = namedtuple( - 'Lease', ['lease_or_host', 'ip', 'hardware', 'ends']) - return Lease(entry_type, ip, Hardware(mac), ends) - - def fake_parsed_host(self, ip=None, mac=None): - """Fake a host declaration as produced by the parser.""" - return self.fake_parsed_lease(ip=ip, mac=mac, entry_type='host') - - def fake_parsed_rubout(self, ip=None): - """Fake a "rubout" host declaration.""" - if ip is None: - ip = factory.getRandomIPAddress() - Rubout = namedtuple('Rubout', ['lease_or_host', 'ip']) - return Rubout('host', ip) - - def test_get_expiry_date_parses_expiry_date(self): - lease = self.fake_parsed_lease(ends='0 2011/01/02 03:04:05') - self.assertEqual( - datetime( - year=2011, month=01, day=02, - hour=03, minute=04, second=05), - get_expiry_date(lease)) - - def test_get_expiry_date_returns_None_for_never(self): - self.assertIsNone( - get_expiry_date(self.fake_parsed_lease(ends='never'))) - - def test_get_expiry_date_returns_None_if_no_expiry_given(self): - self.assertIsNone(get_expiry_date(self.fake_parsed_lease(ends=None))) - - def test_has_expired_returns_False_for_eternal_lease(self): - now = datetime.utcnow() - self.assertFalse(has_expired(self.fake_parsed_lease(ends=None), now)) - - def test_has_expired_returns_False_for_future_expiry_date(self): - now = datetime.utcnow() - later = '1 2035/12/31 23:59:59' - self.assertFalse(has_expired(self.fake_parsed_lease(ends=later), now)) - - def test_has_expired_returns_True_for_past_expiry_date(self): - now = datetime.utcnow() - earlier = '1 2001/01/01 00:00:00' - self.assertTrue( - has_expired(self.fake_parsed_lease(ends=earlier), now)) - - def test_gather_leases_finds_current_leases(self): - lease = self.fake_parsed_lease() - self.assertEqual( - {lease.ip: lease.hardware.mac}, - gather_leases([lease])) - - def test_gather_leases_ignores_expired_leases(self): - earlier = '1 2001/01/01 00:00:00' - lease = self.fake_parsed_lease(ends=earlier) - self.assertEqual({}, gather_leases([lease])) - - def test_gather_leases_combines_expired_and_current_leases(self): - earlier = '1 2001/01/01 00:00:00' - ip = factory.getRandomIPAddress() - old_owner = factory.getRandomMACAddress() - new_owner = factory.getRandomMACAddress() - leases = [ - self.fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), - self.fake_parsed_lease(ip=ip, mac=new_owner), - ] - self.assertEqual({ip: new_owner}, gather_leases(leases)) - - def test_gather_leases_ignores_ordering(self): - earlier = '1 2001/01/01 00:00:00' +def fake_parsed_lease(ip=None, mac=None, ends=None, + entry_type='lease'): + """Fake a lease as produced by the parser.""" + if ip is None: ip = factory.getRandomIPAddress() - old_owner = factory.getRandomMACAddress() - new_owner = factory.getRandomMACAddress() - leases = [ - self.fake_parsed_lease(ip=ip, mac=new_owner), - self.fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), - ] - self.assertEqual({ip: new_owner}, gather_leases(leases)) + if mac is None: + mac = factory.getRandomMACAddress() + Hardware = namedtuple('Hardware', ['mac']) + Lease = namedtuple( + 'Lease', ['lease_or_host', 'ip', 'hardware', 'ends']) + return Lease(entry_type, ip, Hardware(mac), ends) - def test_gather_leases_ignores_host_declarations(self): - self.assertEqual({}, gather_leases([self.fake_parsed_host()])) - def test_gather_hosts_finds_hosts(self): - host = self.fake_parsed_host() - self.assertEqual({host.ip: host.hardware.mac}, gather_hosts([host])) +def fake_parsed_host(ip=None, mac=None): + """Fake a host declaration as produced by the parser.""" + return fake_parsed_lease(ip=ip, mac=mac, entry_type='host') - def test_gather_hosts_ignores_unaccompanied_rubouts(self): - self.assertEqual({}, gather_hosts([self.fake_parsed_rubout()])) - def test_gather_hosts_ignores_rubbed_out_entries(self): +def fake_parsed_rubout(ip=None): + """Fake a "rubout" host declaration.""" + if ip is None: ip = factory.getRandomIPAddress() - hosts = [ - self.fake_parsed_host(ip=ip), - self.fake_parsed_rubout(ip=ip), - ] - self.assertEqual({}, gather_hosts(hosts)) + Rubout = namedtuple('Rubout', ['lease_or_host', 'ip']) + return Rubout('host', ip) - def test_gather_hosts_follows_reassigned_host(self): - ip = factory.getRandomIPAddress() - new_owner = factory.getRandomMACAddress() - hosts = [ - self.fake_parsed_host(ip=ip), - self.fake_parsed_rubout(ip=ip), - self.fake_parsed_host(ip=ip, mac=new_owner), - ] - self.assertEqual({ip: new_owner}, gather_hosts(hosts)) - def test_is_lease_and_is_host_recognize_lease(self): - params = { - 'ip': factory.getRandomIPAddress(), - 'mac': factory.getRandomMACAddress(), - } - [parsed_lease] = lease_parser.searchString(dedent("""\ - lease %(ip)s { - hardware ethernet %(mac)s; - } - """ % params)) - self.assertEqual( - (True, False), - (is_lease(parsed_lease), is_host(parsed_lease))) - - def test_is_lease_and_is_host_recognize_host(self): - params = { - 'ip': factory.getRandomIPAddress(), - 'mac': factory.getRandomMACAddress(), - } - [parsed_host] = lease_parser.searchString(dedent("""\ - host %(ip)s { - hardware ethernet %(mac)s; - } - """ % params)) - self.assertEqual( - (False, True), - (is_lease(parsed_host), is_host(parsed_host))) +class TestLeasesParsers(MAASTestCase): - def test_get_host_mac_returns_None_for_host(self): - params = { - 'ip': factory.getRandomIPAddress(), - 'mac': factory.getRandomMACAddress(), - } - [parsed_host] = lease_parser.searchString(dedent("""\ - host %(ip)s { - hardware ethernet %(mac)s; - } - """ % params)) - self.assertEqual(params['mac'], get_host_mac(parsed_host)) - - def test_get_host_mac_returns_None_for_rubout(self): - ip = factory.getRandomIPAddress() - [parsed_host] = lease_parser.searchString(dedent("""\ - host %s { - deleted; - } - """ % ip)) - self.assertIsNone(get_host_mac(parsed_host)) - - def test_get_host_mac_returns_None_for_rubout_even_with_mac(self): - params = { - 'ip': factory.getRandomIPAddress(), - 'mac': factory.getRandomMACAddress(), - } - [parsed_host] = lease_parser.searchString(dedent("""\ - host %(ip)s { - deleted; - hardware ethernet %(mac)s; - } - """ % params)) - self.assertIsNone(get_host_mac(parsed_host)) + scenarios = ( + ("original", dict(parse=leases_parser.parse_leases)), + ("fast", dict(parse=leases_parser_fast.parse_leases)), + ) def test_parse_leases_copes_with_empty_file(self): - self.assertEqual({}, parse_leases("")) + self.assertEqual({}, self.parse("")) def test_parse_leases_parses_lease(self): params = { 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ lease %(ip)s { starts 5 2010/01/01 00:00:01; ends never; @@ -254,7 +112,7 @@ 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ host %(ip)s { dynamic; hardware ethernet %(mac)s; @@ -263,8 +121,36 @@ """ % params)) self.assertEqual({params['ip']: params['mac']}, leases) + def test_parse_leases_copes_with_misleading_values(self): + params = { + 'ip1': factory.getRandomIPAddress(), + 'mac1': factory.getRandomMACAddress(), + 'ip2': factory.getRandomIPAddress(), + 'mac2': factory.getRandomMACAddress(), + } + leases = self.parse(dedent("""\ + host %(ip1)s { + dynamic; + ### NOTE the following value has a closing brace, and + ### also looks like a host record. + uid "foo}host 12.34.56.78 { }"; + hardware ethernet %(mac1)s; + fixed-address %(ip1)s; + } + ### NOTE the extra indent on the line below. + host %(ip2)s { + dynamic; + hardware ethernet %(mac2)s; + fixed-address %(ip2)s; + } + """ % params)) + self.assertEqual( + {params['ip1']: params['mac1'], + params['ip2']: params['mac2']}, + leases) + def test_parse_leases_parses_host_rubout(self): - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ host %s { deleted; } @@ -277,7 +163,7 @@ 'mac': factory.getRandomMACAddress(), 'incomplete_ip': factory.getRandomIPAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ lease %(ip)s { hardware ethernet %(mac)s; } @@ -291,7 +177,7 @@ 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ # Top comment (ignored). lease %(ip)s { # End-of-line comment (ignored). # Comment in lease block (ignored). @@ -306,7 +192,7 @@ 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ lease %(ip)s { hardware ethernet %(mac)s; ends 1 2001/01/01 00:00:00; @@ -319,7 +205,7 @@ 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ lease %(ip)s { hardware ethernet %(mac)s; ends never; @@ -332,7 +218,7 @@ 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ lease %(ip)s { hardware ethernet %(mac)s; } @@ -345,7 +231,7 @@ 'old_owner': factory.getRandomMACAddress(), 'new_owner': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ lease %(ip)s { hardware ethernet %(old_owner)s; } @@ -360,7 +246,7 @@ 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ host %(ip)s { dynamic; hardware ethernet %(mac)s; @@ -375,7 +261,7 @@ 'ip': factory.getRandomIPAddress(), 'mac': factory.getRandomMACAddress(), } - leases = parse_leases(dedent("""\ + leases = self.parse(dedent("""\ host %(ip)s { hardware ethernet %(mac)s; fixed-address %(ip)s; @@ -383,13 +269,235 @@ """ % params)) self.assertEqual({params['ip']: params['mac']}, leases) + +class TestLeasesParserFast(MAASTestCase): + + def test_expired_lease_does_not_shadow_earlier_host_stanza(self): + params = { + 'ip': factory.getRandomIPAddress(), + 'mac1': factory.getRandomMACAddress(), + 'mac2': factory.getRandomMACAddress(), + } + leases = leases_parser_fast.parse_leases(dedent("""\ + host %(ip)s { + dynamic; + hardware ethernet %(mac1)s; + fixed-address %(ip)s; + } + lease %(ip)s { + starts 5 2010/01/01 00:00:01; + ends 1 2010/01/01 00:00:02; + hardware ethernet %(mac2)s; + } + """ % params)) + # The lease has expired so it doesn't shadow the host stanza, + # and so the MAC returned is from the host stanza. + self.assertEqual({params["ip"]: params["mac1"]}, leases) + + def test_active_lease_shadows_earlier_host_stanza(self): + params = { + 'ip': factory.getRandomIPAddress(), + 'mac1': factory.getRandomMACAddress(), + 'mac2': factory.getRandomMACAddress(), + } + leases = leases_parser_fast.parse_leases(dedent("""\ + host %(ip)s { + dynamic; + hardware ethernet %(mac1)s; + fixed-address %(ip)s; + } + lease %(ip)s { + starts 5 2010/01/01 00:00:01; + hardware ethernet %(mac2)s; + } + """ % params)) + # The lease hasn't expired, so shadows the earlier host stanza. + self.assertEqual({params["ip"]: params["mac2"]}, leases) + + def test_host_stanza_shadows_earlier_active_lease(self): + params = { + 'ip': factory.getRandomIPAddress(), + 'mac1': factory.getRandomMACAddress(), + 'mac2': factory.getRandomMACAddress(), + } + leases = leases_parser_fast.parse_leases(dedent("""\ + lease %(ip)s { + starts 5 2010/01/01 00:00:01; + hardware ethernet %(mac2)s; + } + host %(ip)s { + dynamic; + hardware ethernet %(mac1)s; + fixed-address %(ip)s; + } + """ % params)) + # The lease hasn't expired, but the host entry is later, so it + # shadows the earlier lease stanza. + self.assertEqual({params["ip"]: params["mac1"]}, leases) + + +class TestLeasesParserFunctions(MAASTestCase): + + def test_get_expiry_date_parses_expiry_date(self): + lease = fake_parsed_lease(ends='0 2011/01/02 03:04:05') + self.assertEqual( + datetime( + year=2011, month=01, day=02, + hour=03, minute=04, second=05), + get_expiry_date(lease)) + + def test_get_expiry_date_returns_None_for_never(self): + self.assertIsNone( + get_expiry_date(fake_parsed_lease(ends='never'))) + + def test_get_expiry_date_returns_None_if_no_expiry_given(self): + self.assertIsNone(get_expiry_date(fake_parsed_lease(ends=None))) + + def test_has_expired_returns_False_for_eternal_lease(self): + now = datetime.utcnow() + self.assertFalse(has_expired(fake_parsed_lease(ends=None), now)) + + def test_has_expired_returns_False_for_future_expiry_date(self): + now = datetime.utcnow() + later = '1 2035/12/31 23:59:59' + self.assertFalse(has_expired(fake_parsed_lease(ends=later), now)) + + def test_has_expired_returns_True_for_past_expiry_date(self): + now = datetime.utcnow() + earlier = '1 2001/01/01 00:00:00' + self.assertTrue( + has_expired(fake_parsed_lease(ends=earlier), now)) + + def test_gather_leases_finds_current_leases(self): + lease = fake_parsed_lease() + self.assertEqual( + {lease.ip: lease.hardware.mac}, + gather_leases([lease])) + + def test_gather_leases_ignores_expired_leases(self): + earlier = '1 2001/01/01 00:00:00' + lease = fake_parsed_lease(ends=earlier) + self.assertEqual({}, gather_leases([lease])) + + def test_gather_leases_combines_expired_and_current_leases(self): + earlier = '1 2001/01/01 00:00:00' + ip = factory.getRandomIPAddress() + old_owner = factory.getRandomMACAddress() + new_owner = factory.getRandomMACAddress() + leases = [ + fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), + fake_parsed_lease(ip=ip, mac=new_owner), + ] + self.assertEqual({ip: new_owner}, gather_leases(leases)) + + def test_gather_leases_ignores_ordering(self): + earlier = '1 2001/01/01 00:00:00' + ip = factory.getRandomIPAddress() + old_owner = factory.getRandomMACAddress() + new_owner = factory.getRandomMACAddress() + leases = [ + fake_parsed_lease(ip=ip, mac=new_owner), + fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), + ] + self.assertEqual({ip: new_owner}, gather_leases(leases)) + + def test_gather_leases_ignores_host_declarations(self): + self.assertEqual({}, gather_leases([fake_parsed_host()])) + + def test_gather_hosts_finds_hosts(self): + host = fake_parsed_host() + self.assertEqual({host.ip: host.hardware.mac}, gather_hosts([host])) + + def test_gather_hosts_ignores_unaccompanied_rubouts(self): + self.assertEqual({}, gather_hosts([fake_parsed_rubout()])) + + def test_gather_hosts_ignores_rubbed_out_entries(self): + ip = factory.getRandomIPAddress() + hosts = [ + fake_parsed_host(ip=ip), + fake_parsed_rubout(ip=ip), + ] + self.assertEqual({}, gather_hosts(hosts)) + + def test_gather_hosts_follows_reassigned_host(self): + ip = factory.getRandomIPAddress() + new_owner = factory.getRandomMACAddress() + hosts = [ + fake_parsed_host(ip=ip), + fake_parsed_rubout(ip=ip), + fake_parsed_host(ip=ip, mac=new_owner), + ] + self.assertEqual({ip: new_owner}, gather_hosts(hosts)) + + def test_is_lease_and_is_host_recognize_lease(self): + params = { + 'ip': factory.getRandomIPAddress(), + 'mac': factory.getRandomMACAddress(), + } + [parsed_lease] = lease_parser.searchString(dedent("""\ + lease %(ip)s { + hardware ethernet %(mac)s; + } + """ % params)) + self.assertEqual( + (True, False), + (is_lease(parsed_lease), is_host(parsed_lease))) + + def test_is_lease_and_is_host_recognize_host(self): + params = { + 'ip': factory.getRandomIPAddress(), + 'mac': factory.getRandomMACAddress(), + } + [parsed_host] = lease_parser.searchString(dedent("""\ + host %(ip)s { + hardware ethernet %(mac)s; + } + """ % params)) + self.assertEqual( + (False, True), + (is_lease(parsed_host), is_host(parsed_host))) + + def test_get_host_mac_returns_None_for_host(self): + params = { + 'ip': factory.getRandomIPAddress(), + 'mac': factory.getRandomMACAddress(), + } + [parsed_host] = lease_parser.searchString(dedent("""\ + host %(ip)s { + hardware ethernet %(mac)s; + } + """ % params)) + self.assertEqual(params['mac'], get_host_mac(parsed_host)) + + def test_get_host_mac_returns_None_for_rubout(self): + ip = factory.getRandomIPAddress() + [parsed_host] = lease_parser.searchString(dedent("""\ + host %s { + deleted; + } + """ % ip)) + self.assertIsNone(get_host_mac(parsed_host)) + + def test_get_host_mac_returns_None_for_rubout_even_with_mac(self): + params = { + 'ip': factory.getRandomIPAddress(), + 'mac': factory.getRandomMACAddress(), + } + [parsed_host] = lease_parser.searchString(dedent("""\ + host %(ip)s { + deleted; + hardware ethernet %(mac)s; + } + """ % params)) + self.assertIsNone(get_host_mac(parsed_host)) + def test_combine_entries_accepts_host_followed_by_expired_lease(self): ip = factory.getRandomIPAddress() mac = factory.getRandomMACAddress() earlier = '1 2001/01/01 00:00:00' entries = [ - self.fake_parsed_host(ip=ip, mac=mac), - self.fake_parsed_lease(ip=ip, ends=earlier), + fake_parsed_host(ip=ip, mac=mac), + fake_parsed_lease(ip=ip, ends=earlier), ] self.assertEqual({ip: mac}, combine_entries(entries)) @@ -398,8 +506,8 @@ mac = factory.getRandomMACAddress() earlier = '1 2001/01/01 00:00:00' entries = [ - self.fake_parsed_lease(ip=ip, ends=earlier), - self.fake_parsed_host(ip=ip, mac=mac), + fake_parsed_lease(ip=ip, ends=earlier), + fake_parsed_host(ip=ip, mac=mac), ] self.assertEqual({ip: mac}, combine_entries(entries)) @@ -407,9 +515,9 @@ ip = factory.getRandomIPAddress() mac = factory.getRandomMACAddress() entries = [ - self.fake_parsed_host(ip=ip), - self.fake_parsed_rubout(ip=ip), - self.fake_parsed_lease(ip=ip, mac=mac), + fake_parsed_host(ip=ip), + fake_parsed_rubout(ip=ip), + fake_parsed_lease(ip=ip, mac=mac), ] self.assertEqual({ip: mac}, combine_entries(entries)) @@ -418,9 +526,9 @@ mac = factory.getRandomMACAddress() earlier = '1 2001/01/01 00:00:00' entries = [ - self.fake_parsed_host(ip=ip), - self.fake_parsed_rubout(ip=ip), - self.fake_parsed_lease(ip=ip, mac=mac, ends=earlier), + fake_parsed_host(ip=ip), + fake_parsed_rubout(ip=ip), + fake_parsed_lease(ip=ip, mac=mac, ends=earlier), ] self.assertEqual({}, combine_entries(entries)) @@ -429,9 +537,9 @@ mac = factory.getRandomMACAddress() earlier = '1 2001/01/01 00:00:00' entries = [ - self.fake_parsed_host(ip=ip), - self.fake_parsed_lease(ip=ip, mac=mac, ends=earlier), - self.fake_parsed_rubout(ip=ip), + fake_parsed_host(ip=ip), + fake_parsed_lease(ip=ip, mac=mac, ends=earlier), + fake_parsed_rubout(ip=ip), ] self.assertEqual({}, combine_entries(entries)) @@ -439,9 +547,9 @@ ip = factory.getRandomIPAddress() mac = factory.getRandomMACAddress() entries = [ - self.fake_parsed_host(ip=ip), - self.fake_parsed_lease(ip=ip, mac=mac), - self.fake_parsed_rubout(ip=ip), + fake_parsed_host(ip=ip), + fake_parsed_lease(ip=ip, mac=mac), + fake_parsed_rubout(ip=ip), ] self.assertEqual({ip: mac}, combine_entries(entries)) @@ -449,8 +557,8 @@ ip = factory.getRandomIPAddress() mac = factory.getRandomMACAddress() entries = [ - self.fake_parsed_host(ip=ip), - self.fake_parsed_rubout(ip=ip), - self.fake_parsed_host(ip=ip, mac=mac), + fake_parsed_host(ip=ip), + fake_parsed_rubout(ip=ip), + fake_parsed_host(ip=ip, mac=mac), ] self.assertEqual({ip: mac}, combine_entries(entries)) diff -Nru maas-1.5+bzr2252/src/provisioningserver/driver/__init__.py maas-1.5.4+bzr2294/src/provisioningserver/driver/__init__.py --- maas-1.5+bzr2252/src/provisioningserver/driver/__init__.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/driver/__init__.py 2014-09-03 14:18:31.000000000 +0000 @@ -134,11 +134,25 @@ Architecture(name="i386/generic", description="i386"), Architecture(name="amd64/generic", description="amd64"), Architecture( + name="arm64/generic", description="arm64/generic", + pxealiases=["arm"]), + Architecture( + name="arm64/xgene-uboot", description="arm64/xgene-uboot", + pxealiases=["arm"]), + Architecture( name="armhf/highbank", description="armhf/highbank", pxealiases=["arm"], kernel_options=["console=ttyAMA0"]), Architecture( name="armhf/generic", description="armhf/generic", pxealiases=["arm"], kernel_options=["console=ttyAMA0"]), + # PPC64EL needs a rootdelay for PowerNV. The disk controller + # in the hardware, takes a little bit longer to come up then + # the initrd wants to wait. Set this to 60 seconds, just to + # give the booting machine enough time. This doesn't slow down + # the booting process, it just increases the timeout. + Architecture( + name="ppc64el/generic", description="ppc64el", + kernel_options=['rootdelay=60']), ] for arch in builtin_architectures: ArchitectureRegistry.register_item(arch.name, arch) diff -Nru maas-1.5+bzr2252/src/provisioningserver/drivers/hardware/mscm.py maas-1.5.4+bzr2294/src/provisioningserver/drivers/hardware/mscm.py --- maas-1.5+bzr2252/src/provisioningserver/drivers/hardware/mscm.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/drivers/hardware/mscm.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,187 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Support for managing nodes via the Moonshot HP iLO Chassis Manager CLI. + +This module provides support for interacting with HP Moonshot iLO Chassis +Management (MSCM) CLI via SSH, and for using that support to allow MAAS to +manage systems via iLO. +""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) +str = None + +__metaclass__ = type +__all__ = [ + 'power_control_mscm', + 'probe_and_enlist_mscm', +] + +import re + +from paramiko import ( + AutoAddPolicy, + SSHClient, + ) +import provisioningserver.custom_hardware.utils as utils + + +cartridge_mapping = { + 'ProLiant Moonshot Cartridge': 'amd64/generic', + 'ProLiant m300 Server Cartridge': 'amd64/generic', + 'ProLiant m350 Server Cartridge': 'amd64/generic', + 'ProLiant m400 Server Cartridge': 'arm64/xgene-uboot', + 'ProLiant m500 Server Cartridge': 'amd64/generic', + 'ProLiant m710 Server Cartridge': 'amd64/generic', + 'ProLiant m800 Server Cartridge': 'armhf/keystone', + 'Default': 'arm64/generic', +} + + +class MSCM_CLI_API(object): + """An API for interacting with the Moonshot iLO CM CLI.""" + + def __init__(self, host, username, password): + """MSCM_CLI_API Constructor.""" + self.host = host + self.username = username + self.password = password + self._ssh = SSHClient() + self._ssh.set_missing_host_key_policy(AutoAddPolicy()) + + def _run_cli_command(self, command): + """Run a single command and return unparsed text from stdout.""" + self._ssh.connect( + self.host, username=self.username, password=self.password) + try: + _, stdout, _ = self._ssh.exec_command(command) + output = stdout.read() + finally: + self._ssh.close() + + return output + + def discover_nodes(self): + """Discover all available nodes. + + Example of stdout from running "show node list": + + 'show node list\r\r\nSlot ID Proc Manufacturer + Architecture Memory Power Health\r\n---- + ----- ---------------------- -------------------- + ------ ----- ------\r\n 01 c1n1 Intel Corporation + x86 Architecture 32 GB On OK \r\n 02 c2n1 + N/A No Asset Information \r\n\r\n' + + The regex 'c\d+n\d' is finding the node_id's c1-45n1-8 + """ + node_list = self._run_cli_command("show node list") + return re.findall(r'c\d+n\d', node_list) + + def get_node_macaddr(self, node_id): + """Get node MAC address(es). + + Example of stdout from running "show node macaddr ": + + 'show node macaddr c1n1\r\r\nSlot ID NIC 1 (Switch A) + NIC 2 (Switch B) NIC 3 (Switch A) NIC 4 (Switch B)\r\n + ---- ----- ----------------- ----------------- ----------------- + -----------------\r\n 1 c1n1 a0:1d:48:b5:04:34 a0:1d:48:b5:04:35 + a0:1d:48:b5:04:36 a0:1d:48:b5:04:37\r\n\r\n\r\n' + + The regex '[\:]'.join(['[0-9A-F]{1,2}'] * 6) is finding + the MAC Addresses for the given node_id. + """ + macs = self._run_cli_command("show node macaddr %s" % node_id) + return re.findall(r':'.join(['[0-9a-f]{2}'] * 6), macs) + + def get_node_arch(self, node_id): + """Get node architecture. + + Example of stdout from running "show node info ": + + 'show node info c1n1\r\r\n\r\nCartridge #1 \r\n Type: Compute\r\n + Manufacturer: HP\r\n Product Name: ProLiant m500 Server Cartridge\r\n' + + Parsing this retrieves 'ProLiant m500 Server Cartridge' + """ + node_detail = self._run_cli_command("show node info %s" % node_id) + cartridge = node_detail.split('Product Name: ')[1].splitlines()[0] + if cartridge in cartridge_mapping: + return cartridge_mapping[cartridge] + else: + return cartridge_mapping['Default'] + + def get_node_power_status(self, node_id): + """Get power state of node (on/off). + + Example of stdout from running "show node power ": + + 'show node power c1n1\r\r\n\r\nCartridge #1\r\n Node #1\r\n + Power State: On\r\n' + + Parsing this retrieves 'On' + """ + power_state = self._run_cli_command("show node power %s" % node_id) + return power_state.split('Power State: ')[1].splitlines()[0] + + def power_node_on(self, node_id): + """Power node on.""" + return self._run_cli_command("set node power on %s" % node_id) + + def power_node_off(self, node_id): + """Power node off.""" + return self._run_cli_command("set node power off force %s" % node_id) + + def configure_node_boot_m2(self, node_id): + """Configure HDD boot for node.""" + return self._run_cli_command("set node boot M.2 %s" % node_id) + + def configure_node_bootonce_pxe(self, node_id): + """Configure PXE boot for node once.""" + return self._run_cli_command("set node bootonce pxe %s" % node_id) + + +def power_control_mscm(host, username, password, node_id, power_change): + """Handle calls from the power template for nodes with a power type + of 'mscm'. + """ + mscm = MSCM_CLI_API(host, username, password) + power_status = mscm.get_node_power_status(node_id) + + if power_change == 'off': + mscm.power_node_off(node_id) + return + + if power_change != 'on': + raise AssertionError('Unexpected maas power mode.') + + if power_status == 'On': + mscm.power_node_off(node_id) + + mscm.configure_node_bootonce_pxe(node_id) + mscm.power_node_on(node_id) + + +def probe_and_enlist_mscm(host, username, password): + """ Extracts all of nodes from mscm, sets all of them to boot via HDD by, + default, sets them to bootonce via PXE, and then enlists them into MAAS. + """ + mscm = MSCM_CLI_API(host, username, password) + nodes = mscm.discover_nodes() + for node_id in nodes: + # Set default boot to HDD + mscm.configure_node_boot_m2(node_id) + params = { + 'power_address': host, + 'power_user': username, + 'power_pass': password, + 'node_id': node_id, + } + arch = mscm.get_node_arch(node_id) + macs = mscm.get_node_macaddr(node_id) + utils.create_node(macs, arch, 'mscm', params) diff -Nru maas-1.5+bzr2252/src/provisioningserver/drivers/hardware/tests/test_mscm.py maas-1.5.4+bzr2294/src/provisioningserver/drivers/hardware/tests/test_mscm.py --- maas-1.5+bzr2252/src/provisioningserver/drivers/hardware/tests/test_mscm.py 1970-01-01 00:00:00.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/drivers/hardware/tests/test_mscm.py 2014-09-03 14:18:31.000000000 +0000 @@ -0,0 +1,259 @@ +# Copyright 2014 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Tests for ``provisioningserver.drivers.hardware.mscm``.""" + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, + ) + +str = None + +__metaclass__ = type +__all__ = [] + +from random import randint +import re +from StringIO import StringIO + +from maastesting.factory import factory +from maastesting.matchers import MockCalledOnceWith +from maastesting.testcase import MAASTestCase +from mock import Mock +from provisioningserver.drivers.hardware.mscm import ( + cartridge_mapping, + MSCM_CLI_API, + power_control_mscm, + probe_and_enlist_mscm, + ) +import provisioningserver.custom_hardware.utils as utils + + +def make_mscm_api(): + """Make a MSCM_CLI_API object with randomized parameters.""" + host = factory.make_hostname('mscm') + username = factory.make_name('user') + password = factory.make_name('password') + return MSCM_CLI_API(host, username, password) + + +def make_node_id(): + """Make a node_id.""" + return 'c%sn%s' % (randint(1, 45), randint(1, 8)) + + +def make_show_node_list(length=10): + """Make a fake return value for discover_nodes.""" + return re.findall(r'c\d+n\d', ''.join(make_node_id() + for _ in xrange(length))) + + +def make_show_node_macaddr(length=10): + """Make a fake return value for get_node_macaddr.""" + return ''.join((factory.getRandomMACAddress() + ' ') + for _ in xrange(length)) + + +class TestRunCliCommand(MAASTestCase): + """Tests for ``MSCM_CLI_API.run_cli_command``.""" + + def test_returns_output(self): + api = make_mscm_api() + ssh_mock = self.patch(api, '_ssh') + expected = factory.make_name('output') + stdout = StringIO(expected) + streams = factory.make_streams(stdout=stdout) + ssh_mock.exec_command = Mock(return_value=streams) + output = api._run_cli_command(factory.make_name('command')) + self.assertEqual(expected, output) + + def test_connects_and_closes_ssh_client(self): + api = make_mscm_api() + ssh_mock = self.patch(api, '_ssh') + ssh_mock.exec_command = Mock(return_value=factory.make_streams()) + api._run_cli_command(factory.make_name('command')) + self.assertThat( + ssh_mock.connect, + MockCalledOnceWith( + api.host, username=api.username, password=api.password)) + self.assertThat(ssh_mock.close, MockCalledOnceWith()) + + def test_closes_when_exception_raised(self): + api = make_mscm_api() + ssh_mock = self.patch(api, '_ssh') + + def fail(): + raise Exception('fail') + + ssh_mock.exec_command = Mock(side_effect=fail) + command = factory.make_name('command') + self.assertRaises(Exception, api._run_cli_command, command) + self.assertThat(ssh_mock.close, MockCalledOnceWith()) + + +class TestDiscoverNodes(MAASTestCase): + """Tests for ``MSCM_CLI_API.discover_nodes``.""" + + def test_discover_nodes(self): + api = make_mscm_api() + ssh_mock = self.patch(api, '_ssh') + expected = make_show_node_list() + stdout = StringIO(expected) + streams = factory.make_streams(stdout=stdout) + ssh_mock.exec_command = Mock(return_value=streams) + output = api.discover_nodes() + self.assertEqual(expected, output) + + +class TestNodeMACAddress(MAASTestCase): + """Tests for ``MSCM_CLI_API.get_node_macaddr``.""" + + def test_get_node_macaddr(self): + api = make_mscm_api() + expected = make_show_node_macaddr() + cli_mock = self.patch(api, '_run_cli_command') + cli_mock.return_value = expected + node_id = make_node_id() + output = api.get_node_macaddr(node_id) + self.assertEqual(re.findall(r':'.join(['[0-9a-f]{2}'] * 6), + expected), output) + + +class TestNodeArch(MAASTestCase): + """Tests for ``MSCM_CLI_API.get_node_arch``.""" + + def test_get_node_arch(self): + api = make_mscm_api() + expected = '\r\n Product Name: ProLiant Moonshot Cartridge\r\n' + cli_mock = self.patch(api, '_run_cli_command') + cli_mock.return_value = expected + node_id = make_node_id() + output = api.get_node_arch(node_id) + key = expected.split('Product Name: ')[1].splitlines()[0] + self.assertEqual(cartridge_mapping[key], output) + + +class TestGetNodePowerStatus(MAASTestCase): + """Tests for ``MSCM_CLI_API.get_node_power_status``.""" + + def test_get_node_power_status(self): + api = make_mscm_api() + expected = '\r\n Node #1\r\n Power State: On\r\n' + cli_mock = self.patch(api, '_run_cli_command') + cli_mock.return_value = expected + node_id = make_node_id() + output = api.get_node_power_status(node_id) + self.assertEqual(expected.split('Power State: ')[1].splitlines()[0], + output) + + +class TestPowerAndConfigureNode(MAASTestCase): + """Tests for ``MSCM_CLI_API.configure_node_bootonce_pxe, + MSCM_CLI_API.power_node_on, and MSCM_CLI_API.power_node_off``. + """ + + scenarios = [ + ('power_node_on()', + dict(method='power_node_on')), + ('power_node_off()', + dict(method='power_node_off')), + ('configure_node_bootonce_pxe()', + dict(method='configure_node_bootonce_pxe')), + ] + + def test_returns_expected_outout(self): + api = make_mscm_api() + ssh_mock = self.patch(api, '_ssh') + expected = factory.make_name('output') + stdout = StringIO(expected) + streams = factory.make_streams(stdout=stdout) + ssh_mock.exec_command = Mock(return_value=streams) + output = getattr(api, self.method)(make_node_id()) + self.assertEqual(expected, output) + + +class TestPowerControlMSCM(MAASTestCase): + """Tests for ``power_control_ucsm``.""" + + def test_power_control_mscm_on_on(self): + # power_change and power_status are both 'on' + host = factory.make_hostname('mscm') + username = factory.make_name('user') + password = factory.make_name('password') + node_id = make_node_id() + bootonce_mock = self.patch(MSCM_CLI_API, 'configure_node_bootonce_pxe') + power_status_mock = self.patch(MSCM_CLI_API, 'get_node_power_status') + power_status_mock.return_value = 'On' + power_node_on_mock = self.patch(MSCM_CLI_API, 'power_node_on') + power_node_off_mock = self.patch(MSCM_CLI_API, 'power_node_off') + + power_control_mscm(host, username, password, node_id, + power_change='on') + self.assertThat(bootonce_mock, MockCalledOnceWith(node_id)) + self.assertThat(power_node_off_mock, MockCalledOnceWith(node_id)) + self.assertThat(power_node_on_mock, MockCalledOnceWith(node_id)) + + def test_power_control_mscm_on_off(self): + # power_change is 'on' and power_status is 'off' + host = factory.make_hostname('mscm') + username = factory.make_name('user') + password = factory.make_name('password') + node_id = make_node_id() + bootonce_mock = self.patch(MSCM_CLI_API, 'configure_node_bootonce_pxe') + power_status_mock = self.patch(MSCM_CLI_API, 'get_node_power_status') + power_status_mock.return_value = 'Off' + power_node_on_mock = self.patch(MSCM_CLI_API, 'power_node_on') + + power_control_mscm(host, username, password, node_id, + power_change='on') + self.assertThat(bootonce_mock, MockCalledOnceWith(node_id)) + self.assertThat(power_node_on_mock, MockCalledOnceWith(node_id)) + + def test_power_control_mscm_off_on(self): + # power_change is 'off' and power_status is 'on' + host = factory.make_hostname('mscm') + username = factory.make_name('user') + password = factory.make_name('password') + node_id = make_node_id() + power_status_mock = self.patch(MSCM_CLI_API, 'get_node_power_status') + power_status_mock.return_value = 'On' + power_node_off_mock = self.patch(MSCM_CLI_API, 'power_node_off') + + power_control_mscm(host, username, password, node_id, + power_change='off') + self.assertThat(power_node_off_mock, MockCalledOnceWith(node_id)) + + +class TestProbeAndEnlistMSCM(MAASTestCase): + """Tests for ``probe_and_enlist_mscm``.""" + + def test_probe_and_enlist(self): + host = factory.make_hostname('mscm') + username = factory.make_name('user') + password = factory.make_name('password') + node_id = make_node_id() + macs = make_show_node_macaddr(4) + arch = 'arm64/xgene-uboot' + discover_nodes_mock = self.patch(MSCM_CLI_API, 'discover_nodes') + discover_nodes_mock.return_value = [node_id] + boot_m2_mock = self.patch(MSCM_CLI_API, 'configure_node_boot_m2') + node_arch_mock = self.patch(MSCM_CLI_API, 'get_node_arch') + node_arch_mock.return_value = arch + node_macs_mock = self.patch(MSCM_CLI_API, 'get_node_macaddr') + node_macs_mock.return_value = macs + create_node_mock = self.patch(utils, 'create_node') + probe_and_enlist_mscm(host, username, password) + self.assertThat(discover_nodes_mock, MockCalledOnceWith()) + self.assertThat(boot_m2_mock, MockCalledOnceWith(node_id)) + self.assertThat(node_arch_mock, MockCalledOnceWith(node_id)) + self.assertThat(node_macs_mock, MockCalledOnceWith(node_id)) + params = { + 'power_address': host, + 'power_user': username, + 'power_pass': password, + 'node_id': node_id, + } + self.assertThat(create_node_mock, + MockCalledOnceWith(macs, arch, 'mscm', params)) diff -Nru maas-1.5+bzr2252/src/provisioningserver/power/tests/test_poweraction.py maas-1.5.4+bzr2294/src/provisioningserver/power/tests/test_poweraction.py --- maas-1.5+bzr2252/src/provisioningserver/power/tests/test_poweraction.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/power/tests/test_poweraction.py 2014-09-03 14:18:31.000000000 +0000 @@ -158,20 +158,6 @@ PowerActionFail, pa.execute, power_change='off', mac=factory.getRandomMACAddress()) - def test_virsh_checks_vm_state(self): - # We can't test the virsh template in detail (and it may be - # customized), but by making it use "echo" instead of a real - # virsh we can make it get a bogus answer from its status check. - # The bogus answer is actually the rest of the virsh command - # line. It will complain about this and fail. - action = PowerAction('virsh') - script = action.render_template( - action.get_template(), power_change='on', - power_address='qemu://example.com/', - power_id='mysystem', virsh='echo') - output = action.run_shell(script) - self.assertIn("Got unknown power state from virsh", output) - def test_fence_cdu_checks_state(self): # We can't test the fence_cdu template in detail (and it may be # customized), but by making it use "echo" instead of a real @@ -223,3 +209,25 @@ power_pass='me', power_hwaddress='me', ipmitool='echo') output = action.run_shell(script) self.assertIn("Got unknown power state from ipmipower", output) + + def test_ucsm_renders_template(self): + # I'd like to assert that escape_py_literal is being used here, + # but it's not obvious how to mock things in the template + # rendering namespace so I passed on that. + action = PowerAction('ucsm') + script = action.render_template( + action.get_template(), power_address='foo', + power_user='bar', power_pass='baz', + uuid=factory.getRandomUUID(), power_change='on') + self.assertIn('power_control_ucsm', script) + + def test_mscm_renders_template(self): + # I'd like to assert that escape_py_literal is being used here, + # but it's not obvious how to mock things in the template + # rendering namespace so I passed on that. + action = PowerAction('mscm') + script = action.render_template( + action.get_template(), power_address='foo', + power_user='bar', power_pass='baz', + node_id='c1n1', power_change='on') + self.assertIn('power_control_mscm', script) diff -Nru maas-1.5+bzr2252/src/provisioningserver/power_schema.py maas-1.5.4+bzr2294/src/provisioningserver/power_schema.py --- maas-1.5+bzr2252/src/provisioningserver/power_schema.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/power_schema.py 2014-09-03 14:18:31.000000000 +0000 @@ -168,6 +168,9 @@ 'fields': [ make_json_field('power_address', "Power address"), make_json_field('power_id', "Power ID"), + make_json_field( + 'power_pass', "Power password (optional)", + required=False), ], }, { @@ -226,6 +229,10 @@ make_json_field( 'mac_address', "MAC Address", field_type='mac_address'), make_json_field('power_pass', "Power password"), + make_json_field( + 'power_address', + "An IP address to use instead of the node's primary NIC's IP " + "(i.e. the IP of the MAC above, looked up with ARP)."), ], }, { @@ -238,4 +245,27 @@ make_json_field('power_pass', "Power password"), ], }, + { + 'name': 'ucsm', + 'description': "Cisco UCS Manager", + 'fields': [ + make_json_field('uuid', "Server UUID"), + make_json_field('power_address', "URL for XML API"), + make_json_field('power_user', "API user"), + make_json_field('power_pass', "API password"), + ], + }, + { + 'name': 'mscm', + 'description': "Moonshot HP iLO Chassis Manager", + 'fields': [ + make_json_field('power_address', "IP for MSCM CLI API"), + make_json_field('power_user', "MSCM CLI API user"), + make_json_field('power_pass', "MSCM CLI API password"), + make_json_field( + 'node_id', + "Node ID - Must adhere to cXnY format " + "(X=cartridge number, Y=node number)."), + ], + }, ] diff -Nru maas-1.5+bzr2252/src/provisioningserver/rpc/clusterservice.py maas-1.5.4+bzr2294/src/provisioningserver/rpc/clusterservice.py --- maas-1.5+bzr2252/src/provisioningserver/rpc/clusterservice.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/rpc/clusterservice.py 2014-09-03 14:18:31.000000000 +0000 @@ -221,9 +221,13 @@ instances connected to it. """ + INTERVAL_LOW = 2 # seconds. + INTERVAL_MID = 10 # seconds. + INTERVAL_HIGH = 30 # seconds. + def __init__(self, reactor): super(ClusterClientService, self).__init__( - self._get_random_interval(), self.update) + self._calculate_interval(None, None), self.update) self.connections = {} self.clock = reactor @@ -248,9 +252,6 @@ This obtains a list of endpoints from the region then connects to new ones and drops connections to those no longer used. """ - # 0. Update interval. - self._update_interval() - # 1. Obtain RPC endpoints. try: info_url = self._get_rpc_info_url() info_page = yield getPage(info_url) @@ -258,9 +259,13 @@ eventloops = info["eventloops"] yield self._update_connections(eventloops) except ConnectError as error: + self._update_interval(None, len(self.connections)) log.msg("Region not available: %s" % (error,)) except: + self._update_interval(None, len(self.connections)) log.err() + else: + self._update_interval(len(eventloops), len(self.connections)) @staticmethod def _get_rpc_info_url(): @@ -270,14 +275,39 @@ url = url.geturl() return ascii_url(url) - @staticmethod - def _get_random_interval(): - """Return a random interval between 30 and 90 seconds.""" - return random.randint(30, 90) - - def _update_interval(self): - """Change the interval randomly to avoid stampedes of clusters.""" - self._loop.interval = self.step = self._get_random_interval() + def _calculate_interval(self, num_eventloops, num_connections): + """Calculate the update interval. + + The interval is `INTERVAL_LOW` seconds when there are no + connections, so that this can quickly obtain its first + connection. + + The interval changes to `INTERVAL_MID` seconds when there are + some connections, but fewer than there are event-loops. + + After that it drops back to `INTERVAL_HIGH` seconds. + """ + if num_eventloops is None: + # The region is not available; keep trying regularly. + return self.INTERVAL_LOW + elif num_eventloops == 0: + # The region is coming up; keep trying regularly. + return self.INTERVAL_LOW + elif num_connections == 0: + # No connections to the region; keep trying regularly. + return self.INTERVAL_LOW + elif num_connections < num_eventloops: + # Some connections to the region, but not to all event + # loops; keep updating reasonably frequently. + return self.INTERVAL_MID + else: + # Fully connected to the region; update every so often. + return self.INTERVAL_HIGH + + def _update_interval(self, num_eventloops, num_connections): + """Change the update interval.""" + self._loop.interval = self.step = self._calculate_interval( + num_eventloops, num_connections) @inlineCallbacks def _update_connections(self, eventloops): diff -Nru maas-1.5+bzr2252/src/provisioningserver/rpc/tests/test_clusterservice.py maas-1.5.4+bzr2294/src/provisioningserver/rpc/tests/test_clusterservice.py --- maas-1.5+bzr2252/src/provisioningserver/rpc/tests/test_clusterservice.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/rpc/tests/test_clusterservice.py 2014-09-03 14:18:31.000000000 +0000 @@ -254,40 +254,6 @@ observed_rpc_info_url = ClusterClientService._get_rpc_info_url() self.assertThat(observed_rpc_info_url, Equals(expected_rpc_info_url)) - def test__get_random_interval(self): - # _get_random_interval() returns a random number between 30 and - # 90 inclusive. - is_between_30_and_90_inclusive = MatchesAll( - MatchesAny(GreaterThan(30), Equals(30)), - MatchesAny(LessThan(90), Equals(90))) - for _ in range(100): - self.assertThat( - ClusterClientService._get_random_interval(), - is_between_30_and_90_inclusive) - - def test__get_random_interval_calls_into_standard_library(self): - # _get_random_interval() depends entirely on the standard library. - random = self.patch(clusterservice, "random") - random.randint.return_value = sentinel.randint - self.assertIs( - sentinel.randint, - ClusterClientService._get_random_interval()) - self.assertThat(random.randint, MockCalledOnceWith(30, 90)) - - def test__update_interval(self): - service = ClusterClientService(Clock()) - # ClusterClientService's superclass, TimerService, creates a - # LoopingCall with now=True. We neuter it here because we only - # want to observe the behaviour of _update_interval(). - service.call = (lambda: None, (), {}) - service.startService() - self.assertThat(service.step, MatchesAll( - Equals(service._loop.interval), IsInstance(int))) - service.step = service._loop.interval = sentinel.undefined - service._update_interval() - self.assertThat(service.step, MatchesAll( - Equals(service._loop.interval), IsInstance(int))) - def test_update_connect_error_is_logged_tersely(self): getPage = self.patch(clusterservice, "getPage") getPage.side_effect = error.ConnectionRefusedError() @@ -512,6 +478,53 @@ service.getClient) +class TestClusterClientServiceIntervals(MAASTestCase): + + scenarios = ( + ("initial", { + "num_eventloops": None, + "num_connections": None, + "expected": ClusterClientService.INTERVAL_LOW, + }), + ("no-event-loops", { + "num_eventloops": 0, + "num_connections": sentinel.undefined, + "expected": ClusterClientService.INTERVAL_LOW, + }), + ("no-connections", { + "num_eventloops": 1, # anything > 1. + "num_connections": 0, + "expected": ClusterClientService.INTERVAL_LOW, + }), + ("fewer-connections-than-event-loops", { + "num_eventloops": 2, # anything > num_connections. + "num_connections": 1, # anything > 0. + "expected": ClusterClientService.INTERVAL_MID, + }), + ("default", { + "num_eventloops": 3, # same as num_connections. + "num_connections": 3, # same as num_eventloops. + "expected": ClusterClientService.INTERVAL_HIGH, + }), + ) + + def make_inert_client_service(self): + service = ClusterClientService(Clock()) + # ClusterClientService's superclass, TimerService, creates a + # LoopingCall with now=True. We neuter it here to allow + # observation of the behaviour of _update_interval() for + # example. + service.call = (lambda: None, (), {}) + return service + + def test__calculate_interval(self): + service = self.make_inert_client_service() + service.startService() + self.assertEqual( + self.expected, service._calculate_interval( + self.num_eventloops, self.num_connections)) + + class TestClusterClient(MAASTestCase): run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5) diff -Nru maas-1.5+bzr2252/src/provisioningserver/tasks.py maas-1.5.4+bzr2294/src/provisioningserver/tasks.py --- maas-1.5+bzr2252/src/provisioningserver/tasks.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/tasks.py 2014-09-03 14:18:31.000000000 +0000 @@ -45,6 +45,8 @@ from provisioningserver.custom_hardware.seamicro import ( probe_seamicro15k_and_enlist, ) +from provisioningserver.custom_hardware.ucsm import probe_and_enlist_ucsm +from provisioningserver.custom_hardware.virsh import probe_virsh_and_enlist from provisioningserver.dhcp import ( config, detect, @@ -56,6 +58,7 @@ set_up_options_conf, setup_rndc, ) +from provisioningserver.drivers.hardware.mscm import probe_and_enlist_mscm from provisioningserver.omshell import Omshell from provisioningserver.power.poweraction import ( PowerAction, @@ -383,10 +386,18 @@ call_and_check(['sudo', '-n', 'service', 'maas-dhcp-server', 'restart']) +# Message to put in the DHCP config file when the DHCP server gets stopped. +DISABLED_DHCP_SERVER = "# DHCP server stopped." + + @task @log_exception_text def stop_dhcp_server(): - """Stop a DHCP server.""" + """Write a blank config file and stop a DHCP server.""" + # Write an empty config file to avoid having an outdated config laying + # around. + sudo_write_file( + celery_config.DHCP_CONFIG_FILE, DISABLED_DHCP_SERVER) call_and_check(['sudo', '-n', 'service', 'maas-dhcp-server', 'stop']) @@ -463,7 +474,7 @@ @task @log_exception_text def add_seamicro15k(mac, username, password, power_control=None): - """ See `maasserver.api.NodeGroupsHandler.add_seamicro15k`. """ + """ See `maasserver.api.NodeGroup.add_seamicro15k`. """ ip = find_ip_via_arp(mac) if ip is not None: probe_seamicro15k_and_enlist( @@ -471,3 +482,24 @@ power_control=power_control) else: logger.warning("Couldn't find IP address for MAC %s" % mac) + + +@task +@log_exception_text +def add_virsh(poweraddr, password=None): + """ See `maasserver.api.NodeGroup.add_virsh`. """ + probe_virsh_and_enlist(poweraddr, password=password) + + +@task +@log_exception_text +def enlist_nodes_from_ucsm(url, username, password): + """ See `maasserver.api.NodeGroupHandler.enlist_nodes_from_ucsm`. """ + probe_and_enlist_ucsm(url, username, password) + + +@task +@log_exception_text +def enlist_nodes_from_mscm(host, username, password): + """ See `maasserver.api.NodeGroupHandler.enlist_nodes_from_mscm`. """ + probe_and_enlist_mscm(host, username, password) diff -Nru maas-1.5+bzr2252/src/provisioningserver/tests/test_config.py maas-1.5.4+bzr2294/src/provisioningserver/tests/test_config.py --- maas-1.5+bzr2252/src/provisioningserver/tests/test_config.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/tests/test_config.py 2014-09-03 14:18:31.000000000 +0000 @@ -465,7 +465,7 @@ 'arches': ['i386', 'amd64'], 'release': 'trusty', 'subarches': ['generic'], - 'labels': ['release', 'rc'], + 'labels': ['release'], }, { 'arches': ['i386', 'amd64'], diff -Nru maas-1.5+bzr2252/src/provisioningserver/tests/test_tasks.py maas-1.5.4+bzr2294/src/provisioningserver/tests/test_tasks.py --- maas-1.5+bzr2252/src/provisioningserver/tests/test_tasks.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/tests/test_tasks.py 2014-09-03 14:18:31.000000000 +0000 @@ -73,6 +73,8 @@ from provisioningserver.tags import MissingCredentials from provisioningserver.tasks import ( add_new_dhcp_host_map, + enlist_nodes_from_mscm, + enlist_nodes_from_ucsm, import_boot_images, Omshell, power_off, @@ -329,11 +331,14 @@ self.assertThat(tasks.call_and_check, MockCalledOnceWith( ['sudo', '-n', 'service', 'maas-dhcp-server', 'restart'])) - def test_stop_dhcp_server_sends_command(self): + def test_stop_dhcp_server_sends_command_and_writes_empty_config(self): self.patch(tasks, 'call_and_check') + self.patch(tasks, 'sudo_write_file') stop_dhcp_server() self.assertThat(tasks.call_and_check, MockCalledOnceWith( ['sudo', '-n', 'service', 'maas-dhcp-server', 'stop'])) + self.assertThat(tasks.sudo_write_file, MockCalledOnceWith( + celery_config.DHCP_CONFIG_FILE, tasks.DISABLED_DHCP_SERVER)) def assertTaskRetried(runner, result, nb_retries, task_name): @@ -646,3 +651,25 @@ mock_callback = Mock() import_boot_images(callback=mock_callback) self.assertEqual([call()], mock_callback.delay.mock_calls) + + +class TestAddUCSM(PservTestCase): + + def test_enlist_nodes_from_ucsm(self): + url = 'url' + username = 'username' + password = 'password' + mock = self.patch(tasks, 'probe_and_enlist_ucsm') + enlist_nodes_from_ucsm(url, username, password) + self.assertThat(mock, MockCalledOnceWith(url, username, password)) + + +class TestAddMSCM(PservTestCase): + + def test_enlist_nodes_from_mscm(self): + host = 'host' + username = 'username' + password = 'password' + mock = self.patch(tasks, 'probe_and_enlist_mscm') + enlist_nodes_from_mscm(host, username, password) + self.assertThat(mock, MockCalledOnceWith(host, username, password)) diff -Nru maas-1.5+bzr2252/src/provisioningserver/tests/test_tftp.py maas-1.5.4+bzr2294/src/provisioningserver/tests/test_tftp.py --- maas-1.5+bzr2252/src/provisioningserver/tests/test_tftp.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/tests/test_tftp.py 2014-09-03 14:18:31.000000000 +0000 @@ -28,11 +28,11 @@ from maastesting.testcase import MAASTestCase import mock from provisioningserver import tftp as tftp_module +from provisioningserver.boot import BytesReader from provisioningserver.boot.pxe import PXEBootMethod from provisioningserver.boot.tests.test_pxe import compose_config_path from provisioningserver.tests.test_kernel_opts import make_kernel_parameters from provisioningserver.tftp import ( - BytesReader, TFTPBackend, TFTPService, ) @@ -127,9 +127,9 @@ self.assertEqual(b"", reader.read(1)) @inlineCallbacks - def test_get_reader_config_file(self): - # For paths matching re_config_file, TFTPBackend.get_reader() returns - # a Deferred that will yield a BytesReader. + def test_get_render_file(self): + # For paths matching PXEBootMethod.match_path, TFTPBackend.get_reader() + # returns a Deferred that will yield a BytesReader. cluster_uuid = factory.getRandomUUID() self.patch(tftp_module, 'get_cluster_uuid').return_value = ( cluster_uuid) @@ -147,8 +147,8 @@ factory.getRandomPort()), } - @partial(self.patch, backend, "get_config_reader") - def get_config_reader(boot_method, params): + @partial(self.patch, backend, "get_boot_method_reader") + def get_boot_method_reader(boot_method, params): params_json = json.dumps(params) params_json_reader = BytesReader(params_json) return succeed(params_json_reader) @@ -168,9 +168,10 @@ self.assertEqual(expected_params, observed_params) @inlineCallbacks - def test_get_config_reader_returns_rendered_params(self): - # get_config_reader() takes a dict() of parameters and returns an - # `IReader` of a PXE configuration, rendered by `render_pxe_config`. + def test_get_boot_method_reader_returns_rendered_params(self): + # get_boot_method_reader() takes a dict() of parameters and returns an + # `IReader` of a PXE configuration, rendered by + # `PXEBootMethod.get_reader`. backend = TFTPBackend(self.make_dir(), b"http://example.com/") # Fake configuration parameters, as discovered from the file path. fake_params = {"mac": factory.getRandomMACAddress("-")} @@ -182,15 +183,15 @@ get_page_patch = self.patch(backend, "get_page") get_page_patch.return_value = succeed(fake_get_page_result) - # Stub render_config to return the render parameters. + # Stub get_reader to return the render parameters. method = PXEBootMethod() - fake_render_result = factory.make_name("render") - render_patch = self.patch(method, "render_config") - render_patch.return_value = fake_render_result + fake_render_result = factory.make_name("render").encode("utf-8") + render_patch = self.patch(method, "get_reader") + render_patch.return_value = BytesReader(fake_render_result) # Get the rendered configuration, which will actually be a JSON dump # of the render-time parameters. - reader = yield backend.get_config_reader(method, fake_params) + reader = yield backend.get_boot_method_reader(method, fake_params) self.addCleanup(reader.finish) self.assertIsInstance(reader, BytesReader) output = reader.read(10000) @@ -198,13 +199,13 @@ # The kernel parameters were fetched using `backend.get_page`. self.assertThat(backend.get_page, MockCalledOnceWith(mock.ANY)) - # The result has been rendered by `backend.render_config`. + # The result has been rendered by `method.get_reader`. self.assertEqual(fake_render_result.encode("utf-8"), output) - self.assertThat(method.render_config, MockCalledOnceWith( - kernel_params=fake_kernel_params, **fake_params)) + self.assertThat(method.get_reader, MockCalledOnceWith( + backend, kernel_params=fake_kernel_params, **fake_params)) @inlineCallbacks - def test_get_config_reader_substitutes_armhf_in_params(self): + def test_get_boot_method_render_substitutes_armhf_in_params(self): # get_config_reader() should substitute "arm" for "armhf" in the # arch field of the parameters (mapping from pxe to maas # namespace). @@ -224,8 +225,8 @@ factory.getRandomPort()), } - @partial(self.patch, backend, "get_config_reader") - def get_config_reader(boot_method, params): + @partial(self.patch, backend, "get_boot_method_reader") + def get_boot_method_reader(boot_method, params): params_json = json.dumps(params) params_json_reader = BytesReader(params_json) return succeed(params_json_reader) diff -Nru maas-1.5+bzr2252/src/provisioningserver/tftp.py maas-1.5.4+bzr2294/src/provisioningserver/tftp.py --- maas-1.5+bzr2252/src/provisioningserver/tftp.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/tftp.py 2014-09-03 14:18:31.000000000 +0000 @@ -18,7 +18,6 @@ ] import httplib -from io import BytesIO import json from urllib import urlencode from urlparse import ( @@ -34,10 +33,7 @@ deferred, get_all_interface_addresses, ) -from tftp.backend import ( - FilesystemSynchronousBackend, - IReader, - ) +from tftp.backend import FilesystemSynchronousBackend from tftp.errors import FileNotFound from tftp.protocol import TFTP from twisted.application import internet @@ -45,22 +41,6 @@ from twisted.python.context import get from twisted.web.client import getPage import twisted.web.error -from zope.interface import implementer - - -@implementer(IReader) -class BytesReader: - - def __init__(self, data): - super(BytesReader, self).__init__() - self.buffer = BytesIO(data) - self.size = len(data) - - def read(self, size): - return self.buffer.read(size) - - def finish(self): - self.buffer.close() class TFTPBackend(FilesystemSynchronousBackend): @@ -118,7 +98,7 @@ def get_boot_method(self, file_name): """Finds the correct boot method.""" for _, method in BootMethodRegistry: - params = method.match_config_path(file_name) + params = method.match_path(self, file_name) if params is not None: return method, params return None, None @@ -142,21 +122,19 @@ return d @deferred - def get_config_reader(self, boot_method, params): + def get_boot_method_reader(self, boot_method, params): """Return an `IReader` for a boot method. :param boot_method: Boot method that is generating the config :param params: Parameters so far obtained, typically from the file path requested. """ - def generate_config(kernel_params): - config = boot_method.render_config( - kernel_params=kernel_params, **params) - return config.encode("utf-8") + def generate(kernel_params): + return boot_method.get_reader( + self, kernel_params=kernel_params, **params) d = self.get_kernel_params(params) - d.addCallback(generate_config) - d.addCallback(BytesReader) + d.addCallback(generate) return d @staticmethod @@ -203,7 +181,7 @@ remote_host, remote_port = get("remote", (None, None)) params["remote"] = remote_host params["cluster_uuid"] = get_cluster_uuid() - d = self.get_config_reader(boot_method, params) + d = self.get_boot_method_reader(boot_method, params) d.addErrback(self.get_page_errback, file_name) return d diff -Nru maas-1.5+bzr2252/src/provisioningserver/utils/__init__.py maas-1.5.4+bzr2294/src/provisioningserver/utils/__init__.py --- maas-1.5+bzr2252/src/provisioningserver/utils/__init__.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/utils/__init__.py 2014-09-03 14:18:31.000000000 +0000 @@ -512,6 +512,11 @@ self.__class__.__name__, self.value) +def escape_py_literal(string): + """Escape and quote a string for use as a python literal.""" + return repr(string).decode('ascii') + + class ShellTemplate(tempita.Template): """A Tempita template specialised for writing shell scripts. @@ -824,3 +829,24 @@ if len(columns) == 5 and columns[2] == mac: return columns[0] return None + + +def find_mac_via_arp(ip): + """Find the MAC address for `ip` by reading the output of arp -n. + + Returns `None` if the IP is not found. + + We do this because we aren't necessarily the only DHCP server on the + network, so we can't check our own leases file and be guaranteed to find an + IP that matches. + + :param ip: The ip address, e.g. '192.168.1.1'. + """ + + output = call_capture_and_check(['arp', '-n']).split('\n') + + for line in sorted(output): + columns = line.split() + if len(columns) == 5 and columns[0] == ip: + return columns[2] + return None diff -Nru maas-1.5+bzr2252/src/provisioningserver/utils/tests/test_utils.py maas-1.5.4+bzr2294/src/provisioningserver/utils/tests/test_utils.py --- maas-1.5+bzr2252/src/provisioningserver/utils/tests/test_utils.py 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/src/provisioningserver/utils/tests/test_utils.py 2014-09-03 14:18:31.000000000 +0000 @@ -75,6 +75,7 @@ ExternalProcessError, filter_dict, find_ip_via_arp, + find_mac_via_arp, get_all_interface_addresses, get_mtime, incremental_write, @@ -83,6 +84,7 @@ MainScript, parse_key_value_file, pick_new_mtime, + escape_py_literal, read_text_file, Safe, ShellTemplate, @@ -1321,6 +1323,50 @@ self.assertEqual("192.168.0.1", ip_address_observed) +class TestFindMACViaARP(MAASTestCase): + + def patch_call(self, output): + """Replace `call_capture_and_check` with one that returns `output`.""" + fake = self.patch(provisioningserver.utils, 'call_capture_and_check') + fake.return_value = output + return fake + + def test__resolves_IP_address_to_MAC(self): + sample = """\ + Address HWtype HWaddress Flags Mask Iface + 192.168.100.20 (incomplete) virbr1 + 192.168.0.104 (incomplete) eth0 + 192.168.0.5 (incomplete) eth0 + 192.168.0.2 (incomplete) eth0 + 192.168.0.100 (incomplete) eth0 + 192.168.122.20 ether 52:54:00:02:86:4b C virbr0 + 192.168.0.4 (incomplete) eth0 + 192.168.0.1 ether 90:f6:52:f6:17:92 C eth0 + """ + + call_capture_and_check = self.patch_call(sample) + mac_address_observed = find_mac_via_arp("192.168.122.20") + self.assertThat( + call_capture_and_check, + MockCalledOnceWith(['arp', '-n'])) + self.assertEqual("52:54:00:02:86:4b", mac_address_observed) + + def test__returns_consistent_output(self): + ip = factory.getRandomIPAddress() + macs = [ + '52:54:00:02:86:4b', + '90:f6:52:f6:17:92', + ] + lines = ['%s ether %s C eth0' % (ip, mac) for mac in macs] + self.patch_call('\n'.join(lines)) + one_result = find_mac_via_arp(ip) + self.patch_call('\n'.join(reversed(lines))) + other_result = find_mac_via_arp(ip) + + self.assertIn(one_result, macs) + self.assertEqual(one_result, other_result) + + class TestAsynchronousDecorator(MAASTestCase): run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5) @@ -1372,3 +1418,22 @@ # modification. The arguments passed back match those passed in # from do_stuff_in_thread(). self.assertEqual(((3, 4), {"five": 5}), result) + + +class TestQuotePyLiteral(MAASTestCase): + def test_uses_repr(self): + string = factory.make_name('string') + repr_mock = self.patch(provisioningserver.utils, 'repr') + escape_py_literal(string) + self.assertThat(repr_mock, MockCalledOnceWith(string)) + + def test_decodes_ascii(self): + string = factory.make_name('string') + output = factory.make_name('output') + repr_mock = self.patch(provisioningserver.utils, 'repr') + ascii_value = Mock() + ascii_value.decode = Mock(return_value=output) + repr_mock.return_value = ascii_value + value = escape_py_literal(string) + self.assertThat(ascii_value.decode, MockCalledOnceWith('ascii')) + self.assertEqual(value, output) diff -Nru maas-1.5+bzr2252/versions.cfg maas-1.5.4+bzr2294/versions.cfg --- maas-1.5+bzr2252/versions.cfg 2014-04-15 16:06:09.000000000 +0000 +++ maas-1.5.4+bzr2294/versions.cfg 2014-09-03 14:18:31.000000000 +0000 @@ -37,7 +37,7 @@ nose = 1.3 nose-subunit = 0.2 python-subunit = 0.0.7 -rabbitfixture = 0.3.4 +rabbitfixture = 0.3.5 sst = 0.2.2 testresources = 0.2.5 testscenarios = 0.4