diff -Nru maas-2.3.0-6434-gd354690/debian/changelog maas-2.3.5-6511-gf466fdb/debian/changelog --- maas-2.3.0-6434-gd354690/debian/changelog 2017-11-21 16:53:29.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/debian/changelog 2018-08-23 16:30:04.000000000 +0000 @@ -1,3 +1,19 @@ +maas (2.3.5-6511-gf466fdb-0ubuntu1) xenial-proposed; urgency=medium + + * Stable Release Update. New upstream release, MAAS 2.3.5 (LP: #1772010): + - MAAS 2.3.5 is a new upstream release that fixes a regressions that + affects some users using the MAAS built-in proxy in 2.3.4. + + -- Andres Rodriguez Thu, 23 Aug 2018 12:30:04 -0400 + +maas (2.3.4-6508-g4f77e30-0ubuntu1) xenial-proposed; urgency=medium + + * Stable Release Update. New upstream release, MAAS 2.3.4 (LP: #1772010): + - MAAS 2.3.4 is a new upstream release that fixes several bugs + present on MAAS 2.3.0. + + -- Andres Rodriguez Mon, 30 Jul 2018 20:30:01 -0400 + maas (2.3.0-6434-gd354690-0ubuntu1~16.04.1) xenial-proposed; urgency=medium * Stable Release Update. New upstream release, MAAS 2.3.0 (LP: #1733615): diff -Nru maas-2.3.0-6434-gd354690/requirements.txt maas-2.3.5-6511-gf466fdb/requirements.txt --- maas-2.3.0-6434-gd354690/requirements.txt 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/requirements.txt 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,2 @@ -pyopenssl==16.2.0 pyvmomi==6.0.0.2016.6 -service_identity==16.0.0 git+https://github.com/Supervisor/supervisor@master#egg=supervisor diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/api/interfaces.py maas-2.3.5-6511-gf466fdb/src/maasserver/api/interfaces.py --- maas-2.3.0-6434-gd354690/src/maasserver/api/interfaces.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/api/interfaces.py 2018-08-17 02:41:34.000000000 +0000 @@ -218,6 +218,9 @@ :param bond_xmit_hash_policy: The transmit hash policy to use for slave selection in balance-xor, 802.3ad, and tlb modes. (Default: layer2) + :param bond_num_grat_arp: The number of peer notifications (IPv4 ARP + or IPv6 Neighbour Advertisements) to be issued after a failover. + (Default: 1) Supported bonding modes (bond-mode): balance-rr - Transmit packets in sequential order from the first diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/api/scriptresults.py maas-2.3.5-6511-gf466fdb/src/maasserver/api/scriptresults.py --- maas-2.3.0-6434-gd354690/src/maasserver/api/scriptresults.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/api/scriptresults.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2017 Canonical Ltd. This software is licensed under the +# Copyright 2017-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """API handlers: `ScriptResults`.""" @@ -290,6 +290,18 @@ script_set.delete() return rc.DELETED + def __make_file_title(self, script_result, filetype, extention=None): + title = script_result.name + if extention is not None: + title = '%s.%s' % (title, extention) + if script_result.physical_blockdevice and filetype == 'txt': + title = '%s - /dev/%s' % ( + title, script_result.physical_blockdevice.name) + elif script_result.physical_blockdevice: + title = '%s-%s' % ( + title, script_result.physical_blockdevice.name) + return title + @operation(idempotent=True) def download(self, request, system_id, id): """Download a compressed tar containing all results. @@ -331,32 +343,34 @@ script_set, filters, hardware_type): mtime = time.mktime(script_result.updated.timetuple()) if output == 'combined': - files[script_result.name] = script_result.output - times[script_result.name] = mtime + title = self.__make_file_title(script_result, filetype) + files[title] = script_result.output + times[title] = mtime elif output == 'stdout': - filename = '%s.out' % script_result.name - files[filename] = script_result.stdout - times[filename] = mtime + title = self.__make_file_title(script_result, filetype, 'out') + files[title] = script_result.stdout + times[title] = mtime elif output == 'stderr': - filename = '%s.err' % script_result.name - files[filename] = script_result.stderr - times[filename] = mtime + title = self.__make_file_title(script_result, filetype, 'err') + files[title] = script_result.stderr + times[title] = mtime elif output == 'result': - filename = '%s.yaml' % script_result.name - files[filename] = script_result.result - times[filename] = mtime + title = self.__make_file_title(script_result, filetype, 'yaml') + files[title] = script_result.result + times[title] = mtime elif output == 'all': - files[script_result.name] = script_result.output - times[script_result.name] = mtime - filename = '%s.out' % script_result.name - files[filename] = script_result.stdout - times[filename] = mtime - filename = '%s.err' % script_result.name - files[filename] = script_result.stderr - times[filename] = mtime - filename = '%s.yaml' % script_result.name - files[filename] = script_result.result - times[filename] = mtime + title = self.__make_file_title(script_result, filetype) + files[title] = script_result.output + times[title] = mtime + title = self.__make_file_title(script_result, filetype, 'out') + files[title] = script_result.stdout + times[title] = mtime + title = self.__make_file_title(script_result, filetype, 'err') + files[title] = script_result.stderr + times[title] = mtime + title = self.__make_file_title(script_result, filetype, 'yaml') + files[title] = script_result.result + times[title] = mtime if filetype == 'txt' and len(files) == 1: # Just output the result with no break to allow for piping. diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_enlistment.py maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_enlistment.py --- maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_enlistment.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_enlistment.py 2018-08-17 02:41:34.000000000 +0000 @@ -115,6 +115,7 @@ # Add the default values. power_parameters['power_driver'] = 'LAN_2_0' power_parameters['mac_address'] = '' + power_parameters['power_boot_type'] = 'auto' self.assertEqual(http.client.OK, response.status_code) [machine] = Machine.objects.filter(hostname=hostname) self.assertEqual(power_parameters, machine.power_parameters) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_maas.py maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_maas.py --- maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_maas.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_maas.py 2018-08-17 02:41:34.000000000 +0000 @@ -8,6 +8,7 @@ import http.client import json from operator import itemgetter +import random from django.conf import settings from maasserver.forms.settings import CONFIG_ITEMS_KEYS @@ -292,3 +293,47 @@ }) self.assertEqual(http.client.OK, response.status_code) self.assertTrue(Config.objects.get_config("use_peer_proxy")) + + def test_set_config_boot_images_no_proxy(self): + self.become_admin() + response = self.client.post( + reverse('maas_handler'), { + "op": "set_config", + "name": "boot_images_no_proxy", + "value": True, + }) + self.assertEqual(http.client.OK, response.status_code) + self.assertTrue(Config.objects.get_config("boot_images_no_proxy")) + + +class MAASHandlerAPITestForProxyPort(APITestCase.ForUser): + + scenarios = [ + ('valid-port', { + 'port': random.randint(5300, 65535), 'valid': True}), + ('invalid-port_maas-reserved-range', { + 'port': random.randint(5240, 5270), 'valid': False}), + ('invalid-port_system-services', { + 'port': random.randint(0, 1023), 'valid': False}), + ('invalid-port_out-of-range', { + 'port': random.randint(65536, 70000), 'valid': False}), + ] + + def test_set_config_maas_proxy_port(self): + self.become_admin() + port = self.port + response = self.client.post( + reverse('maas_handler'), { + "op": "set_config", + "name": "maas_proxy_port", + "value": port, + }) + if self.valid: + self.assertEqual(http.client.OK, response.status_code) + self.assertEqual( + port, Config.objects.get_config("maas_proxy_port")) + else: + self.assertEqual( + http.client.BAD_REQUEST, + response.status_code, + response.content) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_scriptresults.py maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_scriptresults.py --- maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_scriptresults.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_scriptresults.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2017 Canonical Ltd. This software is licensed under the +# Copyright 2017-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Tests for the script result API.""" @@ -21,6 +21,7 @@ from maasserver.utils.django_urls import reverse from maasserver.utils.orm import reload_object from metadataserver.enum import ( + HARDWARE_TYPE, HARDWARE_TYPE_CHOICES, RESULT_TYPE_CHOICES, ) @@ -679,3 +680,24 @@ }) self.assertThat(response, HasStatusCode(http.client.OK)) self.assertEquals(script_result.output, response.content) + + def test_download_shows_results_from_all_disks(self): + # Regression test for #LP:1755060 + script = factory.make_Script(hardware_type=HARDWARE_TYPE.STORAGE) + script_set = self.make_scriptset() + script_results = [] + for _ in range(3): + bd = factory.make_PhysicalBlockDevice(node=script_set.node) + script_results.append(factory.make_ScriptResult( + script_set=script_set, script=script, + physical_blockdevice=bd)) + + response = self.client.get( + self.get_script_result_uri(script_set), {'op': 'download'}) + + for script_result in script_results: + title = '%s - /dev/%s' % ( + script_result.name, script_result.physical_blockdevice.name) + title = title.encode() + self.assertIn(title, response.content) + self.assertIn(script_result.output, response.content) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_subnets.py maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_subnets.py --- maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_subnets.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_subnets.py 2018-08-17 02:41:34.000000000 +0000 @@ -12,7 +12,6 @@ from django.conf import settings from maasserver.enum import ( IPADDRESS_TYPE, - IPRANGE_TYPE, NODE_STATUS, RDNS_MODE_CHOICES, ) @@ -443,9 +442,7 @@ boilerplate that creates the requested range, then makes sure the unreserved_ip_ranges API call successfully returns an empty list. """ - factory.make_IPRange( - subnet, first_address, last_address, - type=IPRANGE_TYPE.DYNAMIC) + factory.make_IPRange(subnet, first_address, last_address) response = self.client.get( get_subnet_uri(subnet), {'op': 'unreserved_ip_ranges'}) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_users.py maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_users.py --- maas-2.3.0-6434-gd354690/src/maasserver/api/tests/test_users.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/api/tests/test_users.py 2018-08-17 02:41:34.000000000 +0000 @@ -293,6 +293,17 @@ self.assertIn( b'1 static IP address(es) are still allocated', response.content) + def test_DELETE_user_with_iprange_fails(self): + self.become_admin() + user = factory.make_User() + factory.make_IPRange(user=user) + response = self.client.delete( + reverse('user_handler', args=[user.username])) + self.assertEqual( + http.client.BAD_REQUEST, response.status_code, + response.status_code) + self.assertIn(b'1 IP range(s) are still allocated', response.content) + def test_DELETE_user_with_staticaddress_and_transfer(self): self.become_admin() user = factory.make_User() diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/bootsources.py maas-2.3.5-6511-gf466fdb/src/maasserver/bootsources.py --- maas-2.3.0-6434-gd354690/src/maasserver/bootsources.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/bootsources.py 2018-08-17 02:41:34.000000000 +0000 @@ -12,6 +12,7 @@ import html import os +from urllib.parse import urlparse from maasserver.components import ( discard_persistent_error, @@ -25,6 +26,7 @@ Config, Notification, ) +from maasserver.utils import get_maas_user_agent from maasserver.utils.orm import transactional from maasserver.utils.threads import deferToDatabase from provisioningserver.auth import get_maas_user_gpghome @@ -43,7 +45,6 @@ asynchronous, FOREVER, ) -from provisioningserver.utils.version import get_maas_version_user_agent from requests.exceptions import ConnectionError from twisted.internet.defer import inlineCallbacks @@ -91,7 +92,20 @@ # entire process, including controller refresh. When the region # needs to refresh itself it sends itself results over HTTP to # 127.0.0.1. - env['no_proxy'] = '127.0.0.1,localhost' + no_proxy_hosts = '127.0.0.1,localhost' + # When using a proxy and using an image mirror, we may not want + # to use the proxy to download the images, as they could be + # localted in the local network, hence it makes no sense to use + # it. With this, we add the image mirror localtion(s) to the + # no proxy variable, which ensures MAAS contacts the mirror + # directly instead of through the proxy. + no_proxy = Config.objects.get_config('boot_images_no_proxy') + if no_proxy: + sources = get_boot_sources() + for source in sources: + host = urlparse(source["url"]).netloc.split(':')[0] + no_proxy_hosts = ",".join((no_proxy_hosts, host)) + env['no_proxy'] = no_proxy_hosts return env @@ -246,9 +260,10 @@ with tempdir("keyrings") as keyrings_path: [source] = write_all_keyrings(keyrings_path, [source]) try: + user_agent = yield deferToDatabase(get_maas_user_agent) descriptions = download_all_image_descriptions( [source], - user_agent=get_maas_version_user_agent()) + user_agent=user_agent) except (IOError, ConnectionError) as error: errors.append( "Failed to import images from boot source " diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/compose_preseed.py maas-2.3.5-6511-gf466fdb/src/maasserver/compose_preseed.py --- maas-2.3.0-6434-gd354690/src/maasserver/compose_preseed.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/compose_preseed.py 2018-08-17 02:41:34.000000000 +0000 @@ -41,8 +41,12 @@ if http_proxy and not use_peer_proxy: return http_proxy else: + maas_proxy_port = Config.objects.get_config("maas_proxy_port") + if not maas_proxy_port: + maas_proxy_port = 8000 + url = "http://:%d/" % maas_proxy_port return compose_URL( - "http://:8000/", get_maas_facing_server_host( + url, get_maas_facing_server_host( rack_controller, default_region_ip=default_region_ip)) else: return None @@ -56,6 +60,23 @@ return repo_name.strip().replace(' ', '_').lower() +# LP: #1743966 - If the archive is resigned and has a key, then work around +# this by creating an apt_source that includes the key. +def get_cloud_init_legacy_apt_config_to_inject_key_to_archive(node): + arch = node.split_arch()[0] + archive = PackageRepository.objects.get_default_archive(arch) + apt_sources = {} + apt_sources['apt_sources'] = [] + + if archive.key: + apt_sources['apt_sources'].append({ + 'key': archive.key, + 'source': "deb %s $RELEASE main" % (archive.url), + 'filename': 'lp1743966.list', + }) + return apt_sources + + def get_archive_config(node, preserve_sources=False, default_region_ip=None): arch = node.split_arch()[0] archive = PackageRepository.objects.get_default_archive(arch) @@ -336,6 +357,11 @@ node=node, token=token, base_url=base_url, default_region_ip=default_region_ip)) # Add the system configuration information. + # LP: #1743966 - When deploying precise or trusty, if a custom archive + # with a custom key is used, create a work around to inject the key. + if node.distro_series in ['precise', 'trusty']: + cloud_config.update( + get_cloud_init_legacy_apt_config_to_inject_key_to_archive(node)) cloud_config.update(get_system_info()) apt_proxy = get_apt_proxy( node.get_boot_rack_controller(), default_region_ip=default_region_ip) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/enum.py maas-2.3.5-6511-gf466fdb/src/maasserver/enum.py --- maas-2.3.0-6434-gd354690/src/maasserver/enum.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/enum.py 2018-08-17 02:41:34.000000000 +0000 @@ -693,8 +693,8 @@ BOND_LACP_RATE_CHOICES = ( - (BOND_LACP_RATE.SLOW, BOND_LACP_RATE.SLOW), (BOND_LACP_RATE.FAST, BOND_LACP_RATE.FAST), + (BOND_LACP_RATE.SLOW, BOND_LACP_RATE.SLOW), ) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/__init__.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/__init__.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/__init__.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/__init__.py 2018-08-17 02:41:34.000000000 +0000 @@ -1432,6 +1432,16 @@ enable_http_proxy = get_config_field('enable_http_proxy') use_peer_proxy = get_config_field('use_peer_proxy') http_proxy = get_config_field('http_proxy') + # LP: #1787381 - Fix an issue where the UI is overriding config fields + # that are *only* exposed over the API. + # + # XXX - since the UI for these options has been converted to Angular, + # MAAS no longer automatically creates fields for these based on the + # settings forms. As such, this form doesn't validate against the + # settings form (as the DNSForm would do, for example). As such + # . + # These fields need to be added back once LP: #1787467 is fixed. + # maas_proxy_port = get_config_field('maas_proxy_port') class DNSForm(ConfigForm): diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/interface.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/interface.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/interface.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/interface.py 2018-08-17 02:41:34.000000000 +0000 @@ -422,6 +422,11 @@ bond_updelay = forms.IntegerField(min_value=0, initial=0, required=False) + # Note: we don't need a separate bond_num_unsol_na field, since (as of + # Linux kernel 3.0+) it's actually an alias for the same value. + bond_num_grat_arp = forms.IntegerField( + min_value=0, max_value=255, initial=1, required=False) + bond_lacp_rate = forms.ChoiceField( choices=BOND_LACP_RATE_CHOICES, required=False, initial=BOND_LACP_RATE_CHOICES[0][0], error_messages={ diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/iprange.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/iprange.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/iprange.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/iprange.py 2018-08-17 02:41:34.000000000 +0000 @@ -7,6 +7,8 @@ "IPRangeForm", ] +from django import forms +from django.contrib.auth.models import User from maasserver.forms import MAASModelForm from maasserver.models import Subnet from maasserver.models.iprange import IPRange @@ -15,6 +17,9 @@ class IPRangeForm(MAASModelForm): """IPRange creation/edition form.""" + user = forms.ModelChoiceField( + required=False, queryset=User.objects, to_field_name='username') + class Meta: model = IPRange fields = ( @@ -42,6 +47,8 @@ if subnet is not None: data['subnet'] = subnet.id if request is not None: - data['user'] = request.user.id + data['user'] = request.user.username + elif instance.user and 'user' not in data: + data['user'] = instance.user.username super().__init__( data=data, instance=instance, *args, **kwargs) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/settings.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/settings.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/settings.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/settings.py 2018-08-17 02:41:34.000000000 +0000 @@ -74,6 +74,26 @@ return field +def validate_port(value): + """Raise `ValidationError` when the value is set to a port number. that is + either reserved for known services, or for MAAS services to ensure this + doesn't break MAAS or other applications.""" + msg = "Unable to change port number" + if value > 65535 or value <= 0: + raise ValidationError( + "%s. Port number is not between 0 - 65535." % msg) + if value >= 0 and value <= 1023: + raise ValidationError( + "%s. Port number is reserved for system services." % msg) + # 5240 -> reserved for region HTTP. + # 5241 - 4247 -> reserved for other MAAS services. + # 5248 -> reserved for rack HTTP. + # 5250+ -> reserved for region workers (RPC). + if (value >= 5240 and value <= 5270): + raise ValidationError( + "%s. Port number is reserved for MAAS services." % msg) + + def get_default_usable_osystem(default_osystem): """Return the osystem from the clusters that matches the default_osystem. """ @@ -92,6 +112,13 @@ ] +def make_maas_proxy_port_field(*args, **kwargs): + """Build and return the maas_proxy_port field.""" + return forms.IntegerField( + validators=[validate_port], + **kwargs) + + def make_default_distro_series_field(*args, **kwargs): """Build and return the default_distro_series field.""" default_osystem = Config.objects.get_config('default_osystem') @@ -225,6 +252,17 @@ "downloading boot images.") } }, + 'maas_proxy_port': { + 'default': 8000, + 'form': make_maas_proxy_port_field, + 'form_kwargs': { + 'label': "Port to bind the MAAS built-in proxy (default: 8000)", + 'required': False, + 'help_text': ( + "Defines the port used to bind the built-in proxy. The " + "default port is 8000.") + } + }, 'use_peer_proxy': { 'default': False, 'form': forms.BooleanField, @@ -465,6 +503,25 @@ (IMPORT_RESOURCES_SERVICE_PERIOD.total_seconds() / 60.0)) } }, + 'boot_images_no_proxy': { + 'default': False, + 'form': forms.BooleanField, + 'form_kwargs': { + 'required': False, + 'label': ( + "Set no_proxy with the image repository address when MAAS " + "is behind (or set with) a proxy."), + 'help_text': ( + "By default, when MAAS is behind (and set with) a proxy, it " + "is used to download images from the image repository. In " + "some situations (e.g. when using a local image repository) " + "it doesn't make sense for MAAS to use the proxy to download " + "images because it can access them directly. Setting this " + "option allows MAAS to access the (local) image repository " + "directly by setting the no_proxy variable for the MAAS env " + "with the address of the image repository.") + } + }, 'curtin_verbose': { 'default': False, 'form': forms.BooleanField, diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/subnet.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/subnet.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/subnet.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/subnet.py 2018-08-17 02:41:34.000000000 +0000 @@ -8,6 +8,7 @@ ] from django import forms +from django.core.exceptions import ValidationError from maasserver.enum import RDNS_MODE_CHOICES from maasserver.fields import IPListFormField from maasserver.forms import MAASModelForm @@ -16,6 +17,10 @@ from maasserver.models.vlan import VLAN from maasserver.utils.forms import set_form_error from maasserver.utils.orm import get_one +from netaddr import ( + AddrFormatError, + IPNetwork, +) class SubnetForm(MAASModelForm): @@ -86,6 +91,16 @@ cleaned_data = self._clean_vlan(cleaned_data) return cleaned_data + def clean_cidr(self): + data = self.cleaned_data['cidr'] + try: + network = IPNetwork(data) + if network.prefixlen == 0: + raise ValidationError("Prefix length must be greater than 0.") + except AddrFormatError: + raise ValidationError("Required format: /.") + return data + def _clean_name(self, cleaned_data): name = cleaned_data.get("name", None) instance_name_and_cidr_match = ( diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_interface.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_interface.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_interface.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_interface.py 2018-08-17 02:41:34.000000000 +0000 @@ -789,7 +789,8 @@ "bond_miimon": 100, "bond_downdelay": 0, "bond_updelay": 0, - "bond_lacp_rate": "slow", + "bond_num_grat_arp": 1, + "bond_lacp_rate": "fast", "bond_xmit_hash_policy": "layer2", }, interface.params) @@ -803,6 +804,7 @@ bond_miimon = random.randint(0, 1000) bond_downdelay = random.randint(0, 1000) bond_updelay = random.randint(0, 1000) + bond_num_grat_arp = random.randint(0, 255) bond_lacp_rate = factory.pick_choice(BOND_LACP_RATE_CHOICES) bond_xmit_hash_policy = factory.pick_choice( BOND_XMIT_HASH_POLICY_CHOICES) @@ -818,6 +820,7 @@ 'bond_updelay': bond_updelay, 'bond_lacp_rate': bond_lacp_rate, 'bond_xmit_hash_policy': bond_xmit_hash_policy, + 'bond_num_grat_arp': bond_num_grat_arp, }) self.assertTrue(form.is_valid(), dict(form.errors)) interface = form.save() @@ -828,6 +831,7 @@ "bond_updelay": bond_updelay, "bond_lacp_rate": bond_lacp_rate, "bond_xmit_hash_policy": bond_xmit_hash_policy, + "bond_num_grat_arp": bond_num_grat_arp, }, interface.params) def test__rejects_no_parents(self): diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_iprange.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_iprange.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_iprange.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_iprange.py 2018-08-17 02:41:34.000000000 +0000 @@ -149,3 +149,13 @@ self.assertTrue(form.is_valid(), dict(form.errors)) form.save() self.assertEqual(new_comment, reload_object(iprange).comment) + + def test_update_iprange_user(self): + user = factory.make_User() + subnet = factory.make_ipv4_Subnet_with_IPRanges() + iprange = subnet.get_dynamic_ranges().first() + form = IPRangeForm( + instance=iprange, data={"user": user.username}) + self.assertTrue(form.is_valid(), dict(form.errors)) + form.save() + self.assertEqual(user, reload_object(iprange).user) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_parameters.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_parameters.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_parameters.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_parameters.py 2018-08-17 02:41:34.000000000 +0000 @@ -311,7 +311,7 @@ }, form.errors) def test__input_runtime_validates_min(self): - min_runtime = random.randint(0, 100) + min_runtime = random.randint(1, 100) script = factory.make_Script(parameters={'runtime': { 'type': 'runtime', 'min': min_runtime, diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_subnet.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_subnet.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_subnet.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_subnet.py 2018-08-17 02:41:34.000000000 +0000 @@ -12,17 +12,20 @@ from maasserver.testing.factory import factory from maasserver.testing.testcase import MAASServerTestCase from maasserver.utils.orm import reload_object -from testtools.matchers import MatchesStructure +from testtools.matchers import ( + Equals, + MatchesStructure, +) class TestSubnetForm(MAASServerTestCase): def test__requires_cidr(self): form = SubnetForm({}) - self.assertFalse(form.is_valid(), form.errors) + self.assertFalse(form.is_valid(), dict(form.errors)) self.assertEqual({ "cidr": ["This field is required."], - }, form.errors) + }, dict(form.errors)) def test__rejects_provided_space_on_update(self): space = factory.make_Space() @@ -30,24 +33,54 @@ form = SubnetForm(instance=subnet, data={ "space": space.id }) - self.assertFalse(form.is_valid(), form.errors) + self.assertFalse(form.is_valid(), dict(form.errors)) self.assertEqual({ "space": [ "Spaces may no longer be set on subnets. Set the space on the " "underlying VLAN." - ]}, form.errors) + ]}, dict(form.errors)) def test__rejects_space_on_create(self): space = factory.make_Space() form = SubnetForm({ "space": space.id, "cidr": factory._make_random_network() }) - self.assertFalse(form.is_valid(), form.errors) + self.assertFalse(form.is_valid(), dict(form.errors)) self.assertEqual({ "space": [ "Spaces may no longer be set on subnets. Set the space on the " "underlying VLAN." - ]}, form.errors) + ]}, dict(form.errors)) + + def test__rejects_invalid_cidr(self): + form = SubnetForm({ + 'cidr': 'ten dot zero dot zero dot zero slash zero' + }) + self.assertFalse(form.is_valid(), dict(form.errors)) + self.assertEqual({ + "cidr": [ + "Required format: /." + ]}, dict(form.errors)) + + def test__rejects_ipv4_cidr_with_zero_prefixlen(self): + form = SubnetForm({ + 'cidr': '0.0.0.0/0' + }) + self.assertFalse(form.is_valid(), dict(form.errors)) + self.assertEqual({ + "cidr": [ + "Prefix length must be greater than 0." + ]}, dict(form.errors)) + + def test__rejects_ipv6_cidr_with_zero_prefixlen(self): + form = SubnetForm({ + 'cidr': '::/0' + }) + self.assertFalse(form.is_valid(), dict(form.errors)) + self.assertEqual({ + "cidr": [ + "Prefix length must be greater than 0." + ]}, dict(form.errors)) def test__creates_subnet(self): subnet_name = factory.make_name("subnet") @@ -69,7 +102,7 @@ "gateway_ip": gateway_ip, "dns_servers": ','.join(dns_servers), }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) subnet = form.save() self.assertThat( subnet, MatchesStructure.byEquality( @@ -77,6 +110,14 @@ vlan=vlan, cidr=cidr, gateway_ip=gateway_ip, dns_servers=dns_servers)) + def test__removes_host_bits_and_whitespace(self): + form = SubnetForm({ + "cidr": ' 10.0.0.1/24 ', + }) + self.assertTrue(form.is_valid(), dict(form.errors)) + subnet = form.save() + self.assertThat(subnet.cidr, Equals('10.0.0.0/24')) + def test__creates_subnet_name_equal_to_cidr(self): vlan = factory.make_VLAN() network = factory.make_ip4_or_6_network() @@ -85,7 +126,7 @@ "vlan": vlan.id, "cidr": cidr, }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) subnet = form.save() self.assertThat( subnet, MatchesStructure.byEquality( @@ -97,7 +138,7 @@ form = SubnetForm({ "cidr": cidr, }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) subnet = form.save() self.assertThat( subnet, MatchesStructure.byEquality( @@ -113,7 +154,7 @@ "fabric": fabric.id, "vlan": None, }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) subnet = form.save() self.assertThat( subnet, MatchesStructure.byEquality( @@ -129,7 +170,7 @@ "vid": vlan.vid, "vlan": None, }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) subnet = form.save() self.assertThat( subnet, MatchesStructure.byEquality( @@ -146,7 +187,7 @@ "vid": vlan.vid, "vlan": None, }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) subnet = form.save() self.assertThat( subnet, MatchesStructure.byEquality( @@ -161,10 +202,10 @@ "cidr": cidr, "vid": vlan.vid, }) - self.assertFalse(form.is_valid(), form.errors) + self.assertFalse(form.is_valid(), dict(form.errors)) self.assertEqual({ "vid": ["No VLAN with vid %s in default fabric." % vlan.vid] - }, form.errors) + }, dict(form.errors)) def test__error_for_unknown_vid_in_fabric(self): fabric = factory.make_Fabric() @@ -176,10 +217,10 @@ "fabric": fabric.id, "vid": vlan.vid, }) - self.assertFalse(form.is_valid(), form.errors) + self.assertFalse(form.is_valid(), dict(form.errors)) self.assertEqual({ "vid": ["No VLAN with vid %s in fabric %s." % (vlan.vid, fabric)] - }, form.errors) + }, dict(form.errors)) def test__error_for_vlan_not_in_fabric(self): fabric = factory.make_Fabric() @@ -191,15 +232,15 @@ "fabric": fabric.id, "vlan": vlan.id, }) - self.assertFalse(form.is_valid(), form.errors) + self.assertFalse(form.is_valid(), dict(form.errors)) self.assertEqual({ "vlan": ["VLAN %s is not in fabric %s." % (vlan, fabric)] - }, form.errors) + }, dict(form.errors)) def test__doest_require_vlan_or_cidr_on_update(self): subnet = factory.make_Subnet() form = SubnetForm(instance=subnet, data={}) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) def test__updates_subnet(self): new_name = factory.make_name("subnet") @@ -222,7 +263,7 @@ "gateway_ip": new_gateway_ip, "dns_servers": ','.join(new_dns_servers), }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) form.save() subnet = reload_object(subnet) self.assertThat( @@ -242,7 +283,7 @@ "cidr": new_cidr, "gateway_ip": new_gateway_ip, }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) form.save() subnet = reload_object(subnet) self.assertThat( @@ -257,7 +298,7 @@ form = SubnetForm(instance=subnet, data={ "name": factory.make_name("subnet") }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) form.save() subnet = reload_object(subnet) self.assertEquals(dns_servers, subnet.dns_servers) @@ -268,7 +309,7 @@ form = SubnetForm(instance=subnet, data={ "name": new_name, }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) form.save() subnet = reload_object(subnet) self.assertThat( @@ -282,7 +323,7 @@ "gateway_ip": "", "dns_servers": "", }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) form.save() subnet = reload_object(subnet) self.assertThat( @@ -296,7 +337,7 @@ form = SubnetForm(instance=subnet, data={ "dns_servers": ','.join(dns_servers) }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) form.save() subnet = reload_object(subnet) self.assertEquals(dns_servers, @@ -309,7 +350,7 @@ form = SubnetForm(instance=subnet, data={ "dns_servers": " ".join(dns_servers) }) - self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.is_valid(), dict(form.errors)) form.save() subnet = reload_object(subnet) self.assertEquals(dns_servers, diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_vlan.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_vlan.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/tests/test_vlan.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/tests/test_vlan.py 2018-08-17 02:41:34.000000000 +0000 @@ -13,6 +13,11 @@ from maasserver.testing.factory import factory from maasserver.testing.testcase import MAASServerTestCase from maasserver.utils.orm import reload_object +from testtools import ExpectedException +from testtools.matchers import ( + Equals, + Not, +) class TestVLANForm(MAASServerTestCase): @@ -371,3 +376,63 @@ self.assertEqual(secondary_rack, vlan.primary_rack) self.assertEqual(None, vlan.secondary_rack) self.assertTrue(vlan.dhcp_on) + + +class TestVLANFormFabricModification(MAASServerTestCase): + + def test__cannot_move_vlan_with_overlapping_vid(self): + fabric0 = Fabric.objects.get_default_fabric() + fabric1 = factory.make_Fabric() + fabric1_untagged = fabric1.get_default_vlan() + form = VLANForm(instance=fabric1_untagged, data={ + "fabric": fabric0.id + }) + is_valid = form.is_valid() + self.assertThat(is_valid, Equals(False)) + self.assertThat(dict(form.errors), Equals( + {'__all__': [ + 'A VLAN with the specified VID already ' + 'exists in the destination fabric.' + ]} + )) + with ExpectedException(ValueError): + form.save() + + def test__allows_moving_vlan_to_new_fabric_if_vid_is_unique(self): + fabric0 = Fabric.objects.get_default_fabric() + fabric1 = factory.make_Fabric() + fabric1_untagged = fabric1.get_default_vlan() + form = VLANForm(instance=fabric1_untagged, data={ + "fabric": fabric0.id, + "vid": 10 + }) + is_valid = form.is_valid() + self.assertThat(is_valid, Equals(True)) + form.save() + + def test__deletes_empty_fabrics(self): + fabric0 = Fabric.objects.get_default_fabric() + fabric1 = factory.make_Fabric() + fabric1_untagged = fabric1.get_default_vlan() + form = VLANForm(instance=fabric1_untagged, data={ + "fabric": fabric0.id, + "vid": 10 + }) + is_valid = form.is_valid() + self.assertThat(is_valid, Equals(True)) + form.save() + self.assertThat(reload_object(fabric1), Equals(None)) + + def test__does_not_delete_non_empty_fabrics(self): + fabric0 = Fabric.objects.get_default_fabric() + fabric1 = factory.make_Fabric() + factory.make_VLAN(fabric=fabric1) + fabric1_untagged = fabric1.get_default_vlan() + form = VLANForm(instance=fabric1_untagged, data={ + "fabric": fabric0.id, + "vid": 10 + }) + is_valid = form.is_valid() + form.save() + self.assertThat(is_valid, Equals(True)) + self.assertThat(reload_object(fabric1), Not(Equals(None))) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/forms/vlan.py maas-2.3.5-6511-gf466fdb/src/maasserver/forms/vlan.py --- maas-2.3.0-6434-gd354690/src/maasserver/forms/vlan.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/forms/vlan.py 2018-08-17 02:41:34.000000000 +0000 @@ -15,6 +15,7 @@ ) from maasserver.forms import MAASModelForm from maasserver.models import ( + Fabric, RackController, Space, ) @@ -30,6 +31,9 @@ space = SpecifierOrModelChoiceField( queryset=Space.objects.all(), required=False, empty_label="") + fabric = SpecifierOrModelChoiceField( + queryset=Fabric.objects.all(), required=False, empty_label="") + class Meta: model = VLAN fields = ( @@ -42,6 +46,7 @@ 'secondary_rack', 'relay_vlan', 'space', + 'fabric', ) def __init__(self, *args, **kwargs): diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/management/commands/edit_named_options.py maas-2.3.5-6511-gf466fdb/src/maasserver/management/commands/edit_named_options.py --- maas-2.3.0-6434-gd354690/src/maasserver/management/commands/edit_named_options.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/management/commands/edit_named_options.py 2018-08-17 02:41:34.000000000 +0000 @@ -63,10 +63,8 @@ parser.add_argument( '--migrate-conflicting-options', default=False, dest='migrate_conflicting_options', action='store_true', - help="Causes any options that conflict with MAAS-managed options " - "to be deleted from the BIND configuration and moved to the " - "MAAS-managed configuration. Requires the MAAS database to " - "be configured and running.") + help="**This option is now deprecated**. It no longer has any " + "effect and it may be removed in a future release.") def read_file(self, config_path): """Open the named file and return its contents as a string.""" @@ -113,27 +111,36 @@ if 'forwarders' in options_block: bind_forwarders = options_block['forwarders'] + delete_forwarders = False if not dry_run: - config, created = Config.objects.get_or_create( - name='upstream_dns', - defaults={'value': ' '.join(bind_forwarders)}) - if not created: - # A configuration value already exists, so add the - # additional values we found in the configuration file to - # MAAS. - if config.value is None: - config.value = '' - maas_forwarders = OrderedDict.fromkeys( - config.value.split()) - maas_forwarders.update(bind_forwarders) - config.value = ' '.join(maas_forwarders) - config.save() + try: + config, created = Config.objects.get_or_create( + name='upstream_dns', + defaults={'value': ' '.join(bind_forwarders)}) + if not created: + # A configuration value already exists, so add the + # additional values we found in the configuration + # file to MAAS. + if config.value is None: + config.value = '' + maas_forwarders = OrderedDict.fromkeys( + config.value.split()) + maas_forwarders.update(bind_forwarders) + config.value = ' '.join(maas_forwarders) + config.save() + delete_forwarders = True + except: + pass else: stdout.write( "// Append to MAAS forwarders: %s\n" % ' '.join(bind_forwarders)) - del options_block['forwarders'] + # Only delete forwarders from the config if MAAS was able to + # migrate the options. Otherwise leave them in the original + # config. + if delete_forwarders: + del options_block['forwarders'] def migrate_dnssec_validation(self, options_block, dry_run, stdout): """Remove existing dnssec-validation from the options block. @@ -148,19 +155,23 @@ dnssec_validation = options_block['dnssec-validation'] if not dry_run: - config, created = Config.objects.get_or_create( - name='dnssec_validation', - defaults={'value': dnssec_validation}) - if not created: - # Update the MAAS configuration to reflect the new setting - # found in the configuration file. - config.value = dnssec_validation - config.save() + try: + config, created = Config.objects.get_or_create( + name='dnssec_validation', + defaults={'value': dnssec_validation}) + if not created: + # Update the MAAS configuration to reflect the new + # setting found in the configuration file. + config.value = dnssec_validation + config.save() + except: + pass else: stdout.write( "// Set MAAS dnssec_validation to: %s\n" % dnssec_validation) - + # Always attempt to delete this option as MAAS will always create + # a default for it. del options_block['dnssec-validation'] def back_up_existing_file(self, config_path): @@ -199,8 +210,6 @@ stdout = options.get('stdout') if stdout is None: stdout = sys.stdout - migrate_conflicting_options = options.get( - 'migrate_conflicting_options') options_file = self.read_file(config_path) config_dict = self.parse_file(config_path, options_file) @@ -211,9 +220,10 @@ # Modify the configuration (if necessary). self.set_up_include_statement(options_block, config_path) - if migrate_conflicting_options: - self.migrate_forwarders(options_block, dry_run, stdout) - self.migrate_dnssec_validation(options_block, dry_run, stdout) + # Attempt to migrate the conflicting options if there's + # database connection. + self.migrate_forwarders(options_block, dry_run, stdout) + self.migrate_dnssec_validation(options_block, dry_run, stdout) # Re-parse the new configuration, so we can detect any changes. new_content = make_isc_string(config_dict) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/config.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/config.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/config.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/config.py 2018-08-17 02:41:34.000000000 +0000 @@ -69,6 +69,7 @@ 'default_osystem': DEFAULT_OS.name, 'default_distro_series': DEFAULT_OS.get_default_release(), 'enable_http_proxy': True, + 'maas_proxy_port': 8000, 'use_peer_proxy': False, 'http_proxy': None, 'upstream_dns': None, @@ -86,6 +87,7 @@ 'uuid': None, # Images. 'boot_images_auto_import': True, + 'boot_images_no_proxy': False, # Third Party 'enable_third_party_drivers': True, # Disk erasing. diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/interface.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/interface.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/interface.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/interface.py 2018-08-17 02:41:34.000000000 +0000 @@ -1116,6 +1116,8 @@ """Remove all the `IPAddress` link on the interface.""" for ip_address in self.ip_addresses.exclude( alloc_type=IPADDRESS_TYPE.DISCOVERED): + maaslog.info("%s: IP address automatically unlinked: %s" % ( + self.get_log_string(), ip_address)) self.unlink_ip_address(ip_address, clearing_config=clearing_config) def claim_auto_ips(self, exclude_addresses=[]): diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/keysource.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/keysource.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/keysource.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/keysource.py 2018-08-17 02:41:34.000000000 +0000 @@ -54,6 +54,8 @@ auth_id = CharField(max_length=255, null=False, editable=True) + # Whether keys from this source should be automatically updated + # XXX auto-update not implemented yet auto_update = BooleanField(default=False) class Meta(DefaultMeta): @@ -69,5 +71,5 @@ keys = get_protocol_keys(self.protocol, self.auth_id) return [ - SSHKey.objects.create(key=key, user=user, keysource=self) + SSHKey.objects.get_or_create(key=key, user=user, keysource=self)[0] for key in keys] diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/node.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/node.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/node.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/node.py 2018-08-17 02:41:34.000000000 +0000 @@ -1936,7 +1936,7 @@ ) if not user.has_perm(NODE_PERMISSION.EDIT, self): - # You can't enter rescue mode on a node you don't own, + # You can't enter test mode on a node you don't own, # unless you're an admin. raise PermissionDenied() @@ -4141,6 +4141,14 @@ # deal with it. raise + # If the power state cannot be queried(manual power type) transition + # to the previous state right away. + if not self.get_effective_power_info().can_be_queried: + if self.previous_status != NODE_STATUS.DEPLOYED: + self.owner = None + self.status = self.previous_status + self.save() + def _as(self, model): """Create a `model` that shares underlying storage with `self`. diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/partition.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/partition.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/partition.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/partition.py 2018-08-17 02:41:34.000000000 +0000 @@ -179,8 +179,6 @@ def get_partition_number(self): """Return the partition number in the table.""" - # Circular imports. - from maasserver.models.partitiontable import GPT_REQUIRED_SIZE # Sort manually instead of with `order_by`, this will prevent django # from making a query if the partitions are already cached. partitions_in_table = self.partition_table.partitions.all() @@ -190,18 +188,15 @@ # In some instances the first partition is skipped because it # is used by the machine architecture for a specific reason. # * ppc64el - reserved for prep partition - # * amd64 (disk >= 2TiB) - reserved for bios_grub partition + # * amd64 (not UEFI) - reserved for bios_grub partition node = self.get_node() arch, _ = node.split_arch() boot_disk = node.get_boot_disk() bios_boot_method = node.get_bios_boot_method() - if (arch == "ppc64el" and - self.partition_table.block_device.id == boot_disk.id): + block_device = self.partition_table.block_device + if (arch == "ppc64el" and block_device.id == boot_disk.id): return idx + 2 - elif (arch == "amd64" and - self.partition_table.block_device.id == boot_disk.id and - bios_boot_method != "uefi" and - boot_disk.size >= GPT_REQUIRED_SIZE): + elif arch == "amd64" and bios_boot_method != "uefi": return idx + 2 else: return idx + 1 diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/signals/interfaces.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/interfaces.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/signals/interfaces.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/interfaces.py 2018-08-17 02:41:34.000000000 +0000 @@ -52,6 +52,28 @@ log = LegacyLogger() +class InterfaceVisitingThreadLocal(threading.local): + """Since infinite recursion could occur in an arbitrary interface + hierarchy, use thread-local storage to ensure that each interface is only + visited once. + """ + def __init__(self): + super().__init__() + self.visiting = set() + +enabled_or_disabled_thread_local = InterfaceVisitingThreadLocal() + + +def ensure_link_up(interface): + visiting = enabled_or_disabled_thread_local.visiting + if interface.id not in visiting: + try: + visiting.add(interface.id) + interface.ensure_link_up() + finally: + visiting.discard(interface.id) + + def interface_enabled_or_disabled(instance, old_values, **kwargs): """When an interface is enabled be sure at minimum a LINK_UP is created. When an interface is disabled make sure that all its links are removed, @@ -62,9 +84,10 @@ log.msg("%s: Physical interface enabled; ensuring link-up." % ( instance.get_log_string())) # Make sure it has a LINK_UP link, and for its children. - instance.ensure_link_up() + ensure_link_up(instance) for rel in instance.children_relationships.all(): - rel.child.ensure_link_up() + ensure_link_up(rel.child) + else: log.msg("%s: Physical interface disabled; removing links." % ( instance.get_log_string())) @@ -155,16 +178,7 @@ klass, ['params'], delete=False) -class InterfaceUpdateParentsThreadLocal(threading.local): - """Since infinite recursion could occur in an arbitrary interface - hierarchy, use thread-local stroage to ensure that each interface is only - visited once. - """ - def __init__(self): - super().__init__() - self.visiting = set() - -update_parents_thread_local = InterfaceUpdateParentsThreadLocal() +update_parents_thread_local = InterfaceVisitingThreadLocal() def update_interface_parents(sender, instance, created, **kwargs): diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/signals/iprange.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/iprange.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/signals/iprange.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/iprange.py 2018-08-17 02:41:34.000000000 +0000 @@ -7,6 +7,7 @@ "signals", ] +from django.core.exceptions import ObjectDoesNotExist from django.db.models.signals import ( post_delete, post_save, @@ -19,15 +20,29 @@ def post_save_check_range_utilization(sender, instance, created, **kwargs): - if instance.subnet is None: - # Can't find a subnet to complain about. We're done here. + # Be careful when checking for the subnet. In rare cases, such as a + # cascading delete, Django can sometimes pass stale model objects into + # signal handlers, which will raise unexpected DoesNotExist exceptions, + # and/or otherwise invalidate foreign key fields. + # See bug #1702527 for more details. + try: + if instance.subnet is None: + return + except ObjectDoesNotExist: return instance.subnet.update_allocation_notification() def post_delete_check_range_utilization(sender, instance, **kwargs): - if instance.subnet is None: - # Can't find a subnet to complain about. We're done here. + # Be careful when checking for the subnet. In rare cases, such as a + # cascading delete, Django can sometimes pass stale model objects into + # signal handlers, which will raise unexpected DoesNotExist exceptions, + # and/or otherwise invalidate foreign key fields. + # See bug #1702527 for more details. + try: + if instance.subnet is None: + return + except ObjectDoesNotExist: return instance.subnet.update_allocation_notification() diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/signals/staticipaddress.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/staticipaddress.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/signals/staticipaddress.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/staticipaddress.py 2018-08-17 02:41:34.000000000 +0000 @@ -7,6 +7,7 @@ "signals", ] +from django.core.exceptions import ObjectDoesNotExist from django.db.models.signals import ( post_delete, post_init, @@ -141,15 +142,29 @@ def post_save_check_range_utilization(sender, instance, created, **kwargs): - if instance.subnet is None: - # Can't find a subnet to complain about. We're done here. + # Be careful when checking for the subnet. In rare cases, such as a + # cascading delete, Django can sometimes pass stale model objects into + # signal handlers, which will raise unexpected DoesNotExist exceptions, + # and/or otherwise invalidate foreign key fields. + # See bug #1702527 for more details. + try: + if instance.subnet is None: + return + except ObjectDoesNotExist: return instance.subnet.update_allocation_notification() def post_delete_check_range_utilization(sender, instance, **kwargs): - if instance.subnet is None: - # Can't find a subnet to complain about. We're done here. + # Be careful when checking for the subnet. In rare cases, such as a + # cascading delete, Django can sometimes pass stale model objects into + # signal handlers, which will raise unexpected DoesNotExist exceptions, + # and/or otherwise invalidate foreign key fields. + # See bug #1702527 for more details. + try: + if instance.subnet is None: + return + except ObjectDoesNotExist: return instance.subnet.update_allocation_notification() diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/signals/tests/test_interfaces.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/tests/test_interfaces.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/signals/tests/test_interfaces.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/signals/tests/test_interfaces.py 2018-08-17 02:41:34.000000000 +0000 @@ -14,12 +14,18 @@ IPADDRESS_TYPE, NODE_TYPE, ) -from maasserver.models import Controller +from maasserver.models import ( + Controller, + Interface, +) from maasserver.models.config import ( Config, NetworkDiscoveryConfig, ) -from maasserver.models.signals.interfaces import update_parents_thread_local +from maasserver.models.signals.interfaces import ( + ensure_link_up, + update_parents_thread_local, +) from maasserver.testing.factory import factory from maasserver.testing.testcase import MAASServerTestCase from maasserver.utils.orm import reload_object @@ -30,6 +36,11 @@ ) +def _mock_ensure_link_up(self): + """Mock method to test the 'visited' pattern for recursion prevention.""" + ensure_link_up(self) + + class TestEnableAndDisableInterface(MAASServerTestCase): def test__enable_interface_creates_link_up(self): @@ -52,6 +63,14 @@ alloc_type=IPADDRESS_TYPE.STICKY, ip=None) self.assertIsNotNone(link_ip) + def test__ensure_link_up_only_called_once_per_interface(self): + interface = factory.make_Interface( + INTERFACE_TYPE.PHYSICAL, enabled=False) + interface.enabled = True + self.patch(Interface, 'ensure_link_up', _mock_ensure_link_up) + # This will cause a RecursionError if the code doesn't work. + interface.save() + def test__disable_interface_removes_links(self): interface = factory.make_Interface( INTERFACE_TYPE.PHYSICAL, enabled=True) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/staticipaddress.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/staticipaddress.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/staticipaddress.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/staticipaddress.py 2018-08-17 02:41:34.000000000 +0000 @@ -455,7 +455,7 @@ domain.ttl, %s)""" % default_ttl sql_query = """ - SELECT DISTINCT ON (node.hostname, is_boot, family(staticip.ip)) + SELECT DISTINCT ON (fqdn, is_boot, family) CONCAT(node.hostname, '.', domain.name) AS fqdn, node.system_id, node.node_type, @@ -465,16 +465,38 @@ node.boot_interface_id IS NOT NULL AND ( node.boot_interface_id = interface.id OR - node.boot_interface_id = parent.id + node.boot_interface_id = parent.id OR + node.boot_interface_id = parent_parent.id ), False - ) AS is_boot + ) AS is_boot, + CASE + WHEN interface.type = 'bridge' AND + parent_parent.id = node.boot_interface_id THEN 1 + WHEN interface.type = 'bridge' AND + parent.id = node.boot_interface_id THEN 2 + WHEN interface.type = 'bond' AND + parent.id = node.boot_interface_id THEN 3 + WHEN interface.type = 'physical' AND + interface.id = node.boot_interface_id THEN 4 + WHEN interface.type = 'bond' THEN 5 + WHEN interface.type = 'physical' THEN 6 + WHEN interface.type = 'vlan' THEN 7 + WHEN interface.type = 'alias' THEN 8 + WHEN interface.type = 'unknown' THEN 9 + ELSE 10 + END AS preference, + family(staticip.ip) AS family FROM maasserver_interface AS interface LEFT OUTER JOIN maasserver_interfacerelationship AS rel ON interface.id = rel.child_id LEFT OUTER JOIN maasserver_interface AS parent ON rel.parent_id = parent.id + LEFT OUTER JOIN maasserver_interfacerelationship AS parent_rel ON + parent.id = parent_rel.child_id + LEFT OUTER JOIN maasserver_interface AS parent_parent ON + parent_rel.parent_id = parent_parent.id JOIN maasserver_node AS node ON node.id = interface.node_id JOIN maasserver_domain AS domain ON @@ -512,21 +534,10 @@ staticip.ip IS NOT NULL AND host(staticip.ip) != '' ORDER BY - node.hostname, + fqdn, is_boot DESC, - family(staticip.ip), - CASE - WHEN interface.type = 'bond' AND - parent.id = node.boot_interface_id THEN 1 - WHEN interface.type = 'physical' AND - interface.id = node.boot_interface_id THEN 2 - WHEN interface.type = 'bond' THEN 3 - WHEN interface.type = 'physical' THEN 4 - WHEN interface.type = 'vlan' THEN 5 - WHEN interface.type = 'alias' THEN 6 - WHEN interface.type = 'unknown' THEN 7 - ELSE 8 - END, + family, + preference, /* * We want STICKY and USER_RESERVED addresses to be preferred, * followed by AUTO, DHCP, and finally DISCOVERED. @@ -612,7 +623,7 @@ # we will see all of the boot interfaces before we see any non-boot # interface IPs. See Bug#1584850 for (fqdn, system_id, node_type, ttl, - ip, is_boot) in cursor.fetchall(): + ip, is_boot, preference, family) in cursor.fetchall(): mapping[fqdn].node_type = node_type mapping[fqdn].system_id = system_id mapping[fqdn].ttl = ttl diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/subnet.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/subnet.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/subnet.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/subnet.py 2018-08-17 02:41:34.000000000 +0000 @@ -858,6 +858,9 @@ return None def update_allocation_notification(self): + # Workaround for edge cases in Django. (See bug #1702527.) + if self.id is None: + return ident = "ip_exhaustion__subnet_%d" % self.id # Circular imports. from maasserver.models import Config, Notification diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_keysource.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_keysource.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_keysource.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_keysource.py 2018-08-17 02:41:34.000000000 +0000 @@ -52,6 +52,45 @@ self.assertItemsEqual( returned_sshkeys, SSHKey.objects.filter(keysource=keysource)) + def test_import_keys_source_exists_adds_new_keys(self): + user = factory.make_User() + protocol = random.choice( + [KEYS_PROTOCOL_TYPE.LP, KEYS_PROTOCOL_TYPE.GH]) + auth_id = factory.make_name('auth_id') + keysource = factory.make_KeySource(protocol, auth_id) + keys = get_data('data/test_rsa0.pub') + get_data('data/test_rsa1.pub') + mock_get_protocol_keys = self.patch( + keysource_module, 'get_protocol_keys') + mock_get_protocol_keys.return_value = keys.strip().split('\n') + keysource.import_keys(user) + # Add a new key + keys += get_data('data/test_rsa2.pub') + mock_get_protocol_keys.return_value = keys.strip().split('\n') + returned_sshkeys = keysource.import_keys(user) + self.assertEqual(3, SSHKey.objects.count()) + self.assertCountEqual( + returned_sshkeys, SSHKey.objects.filter(keysource=keysource)) + + def test_import_keys_source_exists_doesnt_remove_keys(self): + user = factory.make_User() + protocol = random.choice( + [KEYS_PROTOCOL_TYPE.LP, KEYS_PROTOCOL_TYPE.GH]) + auth_id = factory.make_name('auth_id') + keysource = factory.make_KeySource(protocol, auth_id) + keys = get_data('data/test_rsa0.pub') + get_data('data/test_rsa1.pub') + mock_get_protocol_keys = self.patch( + keysource_module, 'get_protocol_keys') + mock_get_protocol_keys.return_value = keys.strip().split('\n') + returned_sshkeys = keysource.import_keys(user) + # only return one key + keys = get_data('data/test_rsa0.pub') + mock_get_protocol_keys.return_value = keys.strip().split('\n') + keysource.import_keys(user) + # no key is removed + self.assertEqual(2, SSHKey.objects.count()) + self.assertCountEqual( + returned_sshkeys, SSHKey.objects.filter(keysource=keysource)) + class TestKeySourceManager(MAASServerTestCase): """Testing for the:class:`KeySourceManager` model manager.""" diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_node.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_node.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_node.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_node.py 2018-08-17 02:41:34.000000000 +0000 @@ -4315,6 +4315,19 @@ reload_object(node).status, Equals(NODE_STATUS.EXITING_RESCUE_MODE)) + def test_stop_rescue_mode_manual_power_cycles_node_and_sets_status(self): + node = factory.make_Node( + status=NODE_STATUS.RESCUE_MODE, power_type='manual', + previous_status=NODE_STATUS.DEPLOYED) + admin = factory.make_admin() + mock_node_power_cycle = self.patch(node, '_power_cycle') + node.stop_rescue_mode(admin) + + self.expectThat(mock_node_power_cycle, MockCalledOnceWith()) + self.expectThat( + reload_object(node).status, + Equals(NODE_STATUS.DEPLOYED)) + def test_stop_rescue_mode_logs_and_raises_errors(self): admin = factory.make_admin() node = factory.make_Node( diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_partition.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_partition.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_partition.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_partition.py 2018-08-17 02:41:34.000000000 +0000 @@ -306,6 +306,21 @@ self.expectThat(idx, Equals(partition.get_partition_number())) idx += 1 + def test_get_partition_number_starting_at_2_for_amd64_not_gpt(self): + node = factory.make_Node( + bios_boot_method="pxe", architecture="amd64/generic") + block_device = factory.make_PhysicalBlockDevice( + node=node, + size=(MIN_PARTITION_SIZE * 4) + PARTITION_TABLE_EXTRA_SPACE) + partition_table = factory.make_PartitionTable( + block_device=block_device, table_type=PARTITION_TABLE_TYPE.GPT) + partitions = [ + partition_table.add_partition(size=MIN_BLOCK_DEVICE_SIZE) + for _ in range(4) + ] + for idx, partition in enumerate(partitions, 2): + self.assertEqual(partition.get_partition_number(), idx) + def test_get_partition_number_returns_starting_at_2_for_ppc64el(self): node = factory.make_Node( architecture="ppc64el/generic", bios_boot_method="uefi", diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_staticipaddress.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_staticipaddress.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_staticipaddress.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_staticipaddress.py 2018-08-17 02:41:34.000000000 +0000 @@ -276,7 +276,7 @@ subnet = factory.make_Subnet(cidr=network) factory.make_IPRange( subnet, '192.168.230.1', '192.168.230.254', - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) e = self.assertRaises( StaticIPAddressExhaustion, StaticIPAddress.objects.allocate_new, @@ -821,6 +821,84 @@ node.system_id, 30, {vlanip.ip}, node.node_type)} self.assertEqual(expected_mapping, mapping) + def test_get_hostname_ip_mapping_prefers_bridged_bond_pxe_interface(self): + subnet = factory.make_Subnet( + cidr='10.0.0.0/24', dns_servers=[], gateway_ip='10.0.0.1') + node = factory.make_Node_with_Interface_on_Subnet( + hostname='host', subnet=subnet) + eth0 = node.get_boot_interface() + eth0.name = 'eth0' + eth0.save() + eth1 = factory.make_Interface( + INTERFACE_TYPE.PHYSICAL, node=node, name='eth1', vlan=subnet.vlan) + eth2 = factory.make_Interface( + INTERFACE_TYPE.PHYSICAL, node=node, name='eth2', vlan=subnet.vlan) + node.boot_interface = eth1 + node.save() + bond0 = factory.make_Interface( + INTERFACE_TYPE.BOND, node=node, parents=[eth1, eth2], name='bond0') + br_bond0 = factory.make_Interface( + INTERFACE_TYPE.BRIDGE, parents=[bond0], name='br-bond0') + phy_staticip = factory.make_StaticIPAddress( + alloc_type=IPADDRESS_TYPE.STICKY, interface=eth0, + subnet=subnet, ip='10.0.0.2') + bridge_ip = factory.make_StaticIPAddress( + alloc_type=IPADDRESS_TYPE.STICKY, interface=br_bond0, + subnet=subnet, ip='10.0.0.3') + mapping = StaticIPAddress.objects.get_hostname_ip_mapping( + node.domain) + expected_mapping = { + node.fqdn: HostnameIPMapping( + node.system_id, 30, {bridge_ip.ip}, node.node_type), + "%s.%s" % (eth0.name, node.fqdn): HostnameIPMapping( + node.system_id, 30, {phy_staticip.ip}, node.node_type), + } + self.assertThat(mapping, Equals(expected_mapping)) + + def test_get_hostname_ip_mapping_with_v4_and_v6_and_bridged_bonds(self): + subnet_v4 = factory.make_Subnet( + cidr=str(factory.make_ipv4_network().cidr)) + subnet_v6 = factory.make_Subnet( + cidr='2001:db8::/64') + node = factory.make_Node_with_Interface_on_Subnet( + hostname='host', subnet=subnet_v4) + eth0 = node.get_boot_interface() + eth0.name = 'eth0' + eth0.save() + eth1 = factory.make_Interface( + INTERFACE_TYPE.PHYSICAL, node=node, name='eth1') + eth2 = factory.make_Interface( + INTERFACE_TYPE.PHYSICAL, node=node, name='eth2') + node.boot_interface = eth1 + node.save() + bond0 = factory.make_Interface( + INTERFACE_TYPE.BOND, node=node, parents=[eth1, eth2], name='bond0') + br_bond0 = factory.make_Interface( + INTERFACE_TYPE.BRIDGE, parents=[bond0], name='br-bond0') + phy_staticip_v4 = factory.make_StaticIPAddress( + alloc_type=IPADDRESS_TYPE.STICKY, interface=eth0, + subnet=subnet_v4) + bridge_ip_v4 = factory.make_StaticIPAddress( + alloc_type=IPADDRESS_TYPE.STICKY, interface=br_bond0, + subnet=subnet_v4) + phy_staticip_v6 = factory.make_StaticIPAddress( + alloc_type=IPADDRESS_TYPE.STICKY, interface=eth0, + subnet=subnet_v6) + bridge_ip_v6 = factory.make_StaticIPAddress( + alloc_type=IPADDRESS_TYPE.STICKY, interface=br_bond0, + subnet=subnet_v6) + mapping = StaticIPAddress.objects.get_hostname_ip_mapping( + node.domain) + expected_mapping = { + node.fqdn: HostnameIPMapping( + node.system_id, 30, {bridge_ip_v4.ip, bridge_ip_v6.ip}, + node.node_type), + "%s.%s" % (eth0.name, node.fqdn): HostnameIPMapping( + node.system_id, 30, {phy_staticip_v4.ip, phy_staticip_v6.ip}, + node.node_type), + } + self.assertThat(mapping, Equals(expected_mapping)) + def test_get_hostname_ip_mapping_returns_domain_head_ips(self): parent = factory.make_Domain() name = factory.make_name() diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_subnet.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_subnet.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_subnet.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_subnet.py 2018-08-17 02:41:34.000000000 +0000 @@ -69,6 +69,37 @@ ) +class TestSubnet(MAASServerTestCase): + + def test_can_create_update_and_delete_subnet_with_attached_range(self): + subnet = factory.make_Subnet( + cidr="10.0.0.0/8", gateway_ip=None, dns_servers=[]) + iprange = factory.make_IPRange( + subnet, start_ip="10.0.0.1", end_ip="10.255.255.254") + subnet.description = "foo" + subnet.save() + subnet.delete() + iprange.delete() + + def test_can_create_update_and_delete_subnet_with_assigned_ips(self): + subnet = factory.make_Subnet( + cidr="10.0.0.0/8", gateway_ip=None, dns_servers=[]) + iprange = factory.make_IPRange( + subnet, start_ip="10.0.0.1", end_ip="10.255.255.252") + static_ip = factory.make_StaticIPAddress( + "10.255.255.254", subnet=subnet, + alloc_type=IPADDRESS_TYPE.USER_RESERVED) + static_ip_2 = factory.make_StaticIPAddress( + "10.255.255.253", subnet=subnet, + alloc_type=IPADDRESS_TYPE.USER_RESERVED) + subnet.description = "foo" + subnet.save() + static_ip_2.delete() + subnet.delete() + iprange.delete() + static_ip.delete() + + class CreateCidrTest(MAASServerTestCase): def test_creates_cidr_from_ipv4_strings(self): @@ -1025,7 +1056,7 @@ ip="10.0.0.2", interface=rackif, updated=yesterday) factory.make_IPRange( subnet, start_ip="10.0.0.2", end_ip="10.0.0.2", - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) discovery = subnet.get_least_recently_seen_unknown_neighbour() self.assertThat(discovery.ip, Equals("10.0.0.1")) @@ -1047,7 +1078,7 @@ ip="10.0.0.4", interface=rackif, updated=yesterday) factory.make_IPRange( subnet, start_ip="10.0.0.1", end_ip="10.0.0.2", - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) discovery = subnet.get_least_recently_seen_unknown_neighbour() self.assertThat(discovery.ip, Equals("10.0.0.2")) @@ -1081,7 +1112,7 @@ if not self.managed: factory.make_IPRange( subnet, start_ip=first, end_ip=last, - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) subnet = reload_object(subnet) return subnet @@ -1175,14 +1206,14 @@ managed=False) range1 = factory.make_IPRange( subnet, start_ip='10.0.0.1', end_ip='10.0.0.1', - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) subnet = reload_object(subnet) ip = subnet.get_next_ip_for_allocation() self.assertThat(ip, Equals("10.0.0.1")) range1.delete() factory.make_IPRange( subnet, start_ip='10.0.0.6', end_ip='10.0.0.6', - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) subnet = reload_object(subnet) ip = subnet.get_next_ip_for_allocation() self.assertThat(ip, Equals("10.0.0.6")) @@ -1194,7 +1225,7 @@ managed=False) factory.make_IPRange( subnet, start_ip='10.0.0.3', end_ip='10.0.0.4', - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) subnet = reload_object(subnet) ip = subnet.get_next_ip_for_allocation() self.assertThat(ip, Equals("10.0.0.3")) @@ -1324,7 +1355,7 @@ start_ip=str(IPAddress(range_start)), end_ip=str(IPAddress(range_end)), subnet=self.subnet, - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) else: # Dummy value so we allocate an IP below. range_end = self.ipnetwork.first @@ -1356,7 +1387,7 @@ start_ip=str(IPAddress(range_start)), end_ip=str(IPAddress(range_end)), subnet=self.subnet, - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) ident = 'ip_exhaustion__subnet_%d' % self.subnet.id notification = get_one(Notification.objects.filter(ident=ident)) notification_exists = notification is not None @@ -1375,7 +1406,7 @@ start_ip=str(IPAddress(range_start)), end_ip=str(IPAddress(range_end)), subnet=self.subnet, - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) ident = 'ip_exhaustion__subnet_%d' % self.subnet.id notification = get_one(Notification.objects.filter(ident=ident)) notification_exists = notification is not None @@ -1399,7 +1430,7 @@ start_ip=str(IPAddress(range_start)), end_ip=str(IPAddress(range_end)), subnet=self.subnet, - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) ident = 'ip_exhaustion__subnet_%d' % self.subnet.id notification = get_one(Notification.objects.filter(ident=ident)) notification_exists = notification is not None @@ -1429,7 +1460,7 @@ start_ip=str(IPAddress(range_start)), end_ip=str(IPAddress(range_end)), subnet=self.subnet, - type=IPRANGE_TYPE.RESERVED) + alloc_type=IPRANGE_TYPE.RESERVED) else: # Dummy value so we allocate an IP below. range_end = self.ipnetwork.first diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_userprofile.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_userprofile.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/tests/test_userprofile.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/tests/test_userprofile.py 2018-08-17 02:41:34.000000000 +0000 @@ -9,8 +9,6 @@ from maasserver.exceptions import CannotDeleteUserException from maasserver.models import ( FileStorage, - Node, - StaticIPAddress, UserProfile, ) from maasserver.models.user import ( @@ -20,6 +18,7 @@ ) from maasserver.testing.factory import factory from maasserver.testing.testcase import MAASServerTestCase +from maasserver.utils.orm import reload_object from piston3.models import ( Consumer, Token, @@ -105,24 +104,40 @@ # Cannot delete a user with nodes attached to it. profile = factory.make_User().userprofile factory.make_Node(owner=profile.user) - self.assertRaises(CannotDeleteUserException, profile.delete) + error = self.assertRaises(CannotDeleteUserException, profile.delete) + self.assertIn('1 node(s)', str(error)) def test_delete_attached_static_ip_addresses(self): # Cannot delete a user with static IP address attached to it. profile = factory.make_User().userprofile factory.make_StaticIPAddress(user=profile.user) - self.assertRaises(CannotDeleteUserException, profile.delete) + error = self.assertRaises(CannotDeleteUserException, profile.delete) + self.assertIn('1 static IP address(es)', str(error)) + + def test_delete_attached_iprange(self): + # Cannot delete a user with an IP range attached to it. + profile = factory.make_User().userprofile + factory.make_IPRange(user=profile.user) + error = self.assertRaises(CannotDeleteUserException, profile.delete) + self.assertIn('1 IP range(s)', str(error)) + + def test_delete_attached_multiple_resources(self): + profile = factory.make_User().userprofile + factory.make_Node(owner=profile.user) + factory.make_StaticIPAddress(user=profile.user) + error = self.assertRaises(CannotDeleteUserException, profile.delete) + self.assertIn('1 static IP address(es), 1 node(s)', str(error)) def test_transfer_resources(self): user = factory.make_User() node = factory.make_Node(owner=user) ipaddress = factory.make_StaticIPAddress(user=user) + iprange = factory.make_IPRange(user=user) new_user = factory.make_User() user.userprofile.transfer_resources(new_user) - node = Node.objects.get(id=node.id) - self.assertEqual(node.owner, new_user) - ipaddress = StaticIPAddress.objects.get(id=ipaddress.id) - self.assertEqual(ipaddress.user, new_user) + self.assertEqual(reload_object(node).owner, new_user) + self.assertEqual(reload_object(ipaddress).user, new_user) + self.assertEqual(reload_object(iprange).user, new_user) def test_manager_all_users(self): users = set(factory.make_User() for _ in range(3)) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/userprofile.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/userprofile.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/userprofile.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/userprofile.py 2018-08-17 02:41:34.000000000 +0000 @@ -65,20 +65,21 @@ completed_intro = BooleanField(default=False) def delete(self): - addr_count = self.user.staticipaddress_set.count() - if addr_count: - msg = ( - "User {} cannot be deleted: {} static IP address(es) " - "are still allocated to this user.").format( - self.user.username, addr_count) - raise CannotDeleteUserException(msg) - nb_nodes = self.user.node_set.count() - if nb_nodes: - msg = ( - "User {} cannot be deleted: {} node(s) are still " - "allocated to this user.").format( - self.user.username, nb_nodes) - raise CannotDeleteUserException(msg) + # check owned resources + owned_resources = [ + ('staticipaddress', 'static IP address(es)'), + ('iprange', 'IP range(s)'), + ('node', 'node(s)')] + messages = [] + for attr, title in owned_resources: + count = getattr(self.user, attr + '_set').count() + if count: + messages.append('{} {}'.format(count, title)) + + if messages: + raise CannotDeleteUserException( + 'User {} cannot be deleted: {} are still allocated'.format( + self.user.username, ', '.join(messages))) if self.user.filestorage_set.exists(): self.user.filestorage_set.all().delete() @@ -89,8 +90,8 @@ def transfer_resources(self, new_owner): """Transfer owned resources to another user. - Nodes and static IP addresses owned by the user are transfered to the - new owner. + Nodes, static IP addresses and IP ranges owned by the user are + transfered to the new owner. :param new_owner: the UserProfile to transfer ownership to. :type new_owner: maasserver.models.UserProfile @@ -98,6 +99,7 @@ """ self.user.node_set.update(owner=new_owner) self.user.staticipaddress_set.update(user=new_owner) + self.user.iprange_set.update(user=new_owner) def get_authorisation_tokens(self): """Fetches all the user's OAuth tokens. diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/models/vlan.py maas-2.3.5-6511-gf466fdb/src/maasserver/models/vlan.py --- maas-2.3.0-6434-gd354690/src/maasserver/models/vlan.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/models/vlan.py 2018-08-17 02:41:34.000000000 +0000 @@ -15,6 +15,7 @@ BooleanField, CASCADE, CharField, + Count, deletion, ForeignKey, IntegerField, @@ -243,6 +244,15 @@ subnet.vlan = self.fabric.get_default_vlan() subnet.save() + def unique_error_message(self, model_class, unique_check): + if set(unique_check) == {'vid', 'fabric'}: + return ( + 'A VLAN with the specified VID already exists in the ' + 'destination fabric.' + ) + else: + return super().unique_error_message(model_class, unique_check) + def delete(self): if self.is_fabric_default(): raise ValidationError( @@ -268,3 +278,9 @@ "DHCP server is being used.", ident="dhcp_disabled_all_vlans") super().save(*args, **kwargs) + # Circular dependencies. + from maasserver.models import Fabric + # Delete any now-empty fabrics. + fabrics_with_vlan_count = Fabric.objects.annotate( + vlan_count=Count("vlan")) + fabrics_with_vlan_count.filter(vlan_count=0).delete() diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/node_status.py maas-2.3.5-6511-gf466fdb/src/maasserver/node_status.py --- maas-2.3.0-6434-gd354690/src/maasserver/node_status.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/node_status.py 2018-08-17 02:41:34.000000000 +0000 @@ -310,6 +310,7 @@ NODE_STATUS.COMMISSIONING, NODE_STATUS.DISK_ERASING, NODE_STATUS.ENTERING_RESCUE_MODE, + NODE_STATUS.RESCUE_MODE, NODE_STATUS.TESTING, ] diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/preseed_network.py maas-2.3.5-6511-gf466fdb/src/maasserver/preseed_network.py --- maas-2.3.0-6434-gd354690/src/maasserver/preseed_network.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/preseed_network.py 2018-08-17 02:41:34.000000000 +0000 @@ -52,8 +52,6 @@ """Generate route operation place in `network_config`.""" if version == 1: route_operation = { - "id": route.id, - "type": "route", "destination": route.destination.cidr, "gateway": route.gateway_ip, "metric": route.metric, @@ -99,6 +97,19 @@ else: raise ValueError("Unknown interface type: %s" % self.type) + if version == 2: + routes = self._generate_route_operations( + self.matching_routes, version=version) + if len(routes) > 0: + self.config['routes'] = routes + + def _generate_route_operations(self, matching_routes, version=1): + """Generate all route operations.""" + routes = [] + for route in sorted(matching_routes, key=attrgetter("id")): + routes.append(_generate_route_operation(route, version=version)) + return routes + def _generate_physical_operation(self, version=1): """Generate physical interface operation for `interface` and place in `network_config`.""" @@ -241,7 +252,8 @@ self._set_default_gateway( subnet, v1_subnet_operation if version == 1 else v2_config) - if subnet.dns_servers is not None: + if (subnet.dns_servers is not None and + len(subnet.dns_servers) > 0): v1_subnet_operation["dns_nameservers"] = ( subnet.dns_servers) if "nameservers" not in v2_config: @@ -251,8 +263,16 @@ v2_nameservers["addresses"] = [] v2_nameservers["addresses"].extend( [server for server in subnet.dns_servers]) - self.matching_routes.update( - self._get_matching_routes(subnet)) + matching_subnet_routes = self._get_matching_routes(subnet) + if len(matching_subnet_routes) > 0 and version == 1: + # For the v1 YAML, the list of routes is rendered + # within the context of each subnet. + routes = self._generate_route_operations( + matching_subnet_routes, version=version) + v1_subnet_operation['routes'] = routes + # Keep track of routes which apply to the context of this + # interface for rendering the v2 YAML. + self.matching_routes.update(matching_subnet_routes) if dhcp_type: v1_config.append( {"type": dhcp_type} @@ -396,6 +416,15 @@ # which MAAS uses to keep consistent with bridges. params[key.replace("_", "-")] = ( _get_param_value(value)) + bond_mode = params.get('bond-mode') + if bond_mode is not None: + # Bug #1730626: lacp-rate should only be set on 802.3ad bonds. + if bond_mode != "802.3ad": + params.pop("bond-lacp-rate", None) + # Bug #1730991: these parameters only apply to active-backup mode. + if bond_mode != "active-backup": + params.pop("bond-num-grat-arp", None) + params.pop("bond-num-unsol-na", None) return params def _get_bridge_params(self, version=1): @@ -470,7 +499,6 @@ for name in sorted(get_dns_search_paths()) if name != self.node.domain.name ] - self._generate_route_operations(version=version) self.v1_config.append({ "type": "nameserver", "address": default_dns_servers, @@ -508,16 +536,6 @@ # v2_config.update({"nameservers": nameservers}) self.config = network_config - def _generate_route_operations(self, version=1): - """Generate all route operations.""" - routes = [] - for route in sorted(self.matching_routes, key=attrgetter("id")): - routes.append(_generate_route_operation(route, version=version)) - if version == 1: - self.v1_config.extend(routes) - elif version == 2 and len(routes) > 0: - self.v2_config["routes"] = routes - def compose_curtin_network_config(node, version=1): """Compose the network configuration for curtin.""" diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/preseed.py maas-2.3.5-6511-gf466fdb/src/maasserver/preseed.py --- maas-2.3.0-6434-gd354690/src/maasserver/preseed.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/preseed.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2012-2017 Canonical Ltd. This software is licensed under the +# Copyright 2012-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Preseed generation.""" @@ -433,6 +433,14 @@ raise ClusterUnavailable( "Unable to get RPC connection for rack controller '%s' (%s)" % (rack_controller.hostname, rack_controller.system_id)) + # A matching subarch may be a newer subarch which contains support for + # an older one. e.g Xenial hwe-16.04 will match for ga-16.04. First + # try to find the subarch we are deploying, if that isn't found allow + # a newer version. + for image in images: + if (image['purpose'] == 'xinstall' and + image['subarchitecture'] == subarch): + return image for image in images: if image['purpose'] == 'xinstall': return image @@ -457,7 +465,14 @@ # Per etc/services cluster is opening port 5248 to serve images via HTTP image = get_curtin_image(node) if image['xinstall_type'] == 'squashfs': - return 'cp:///media/root-ro' + # XXX: roaksoax LP: #1739761 - Since the switch to squashfs (and drop + # of iscsi), precise is no longer deployable. To address a squashfs + # image is made available allowing it to be deployed in the + # commissioning ephemeral environment. + if series == 'precise': + url_prepend = "fsimage:" + else: + return 'cp:///media/root-ro' elif image['xinstall_type'] == 'tgz': url_prepend = '' else: diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/preseed_storage.py maas-2.3.5-6511-gf466fdb/src/maasserver/preseed_storage.py --- maas-2.3.0-6434-gd354690/src/maasserver/preseed_storage.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/preseed_storage.py 2018-08-17 02:41:34.000000000 +0000 @@ -9,7 +9,10 @@ from operator import attrgetter -from django.db.models import Sum +from django.db.models import ( + Q, + Sum, +) from maasserver.enum import ( FILESYSTEM_GROUP_TYPE, FILESYSTEM_TYPE, @@ -19,7 +22,6 @@ from maasserver.models.partition import Partition from maasserver.models.partitiontable import ( BIOS_GRUB_PARTITION_SIZE, - GPT_REQUIRED_SIZE, INITIAL_PARTITION_OFFSET, PARTITION_TABLE_EXTRA_SPACE, PREP_PARTITION_SIZE, @@ -35,7 +37,8 @@ def __init__(self, node): self.node = node self.boot_disk = node.get_boot_disk() - self.boot_disk_first_partition = None + self.grub_device_ids = [] + self.boot_first_partitions = [] self.operations = { "disk": [], "partition": [], @@ -53,6 +56,7 @@ # Add all the items to operations. self._add_disk_and_filesystem_group_operations() + self._find_grub_devices() self._add_partition_operations() self._add_format_and_mount_operations() @@ -117,18 +121,13 @@ def _requires_prep_partition(self, block_device): """Return True if block device requires the prep partition.""" arch, _ = self.node.split_arch() - return ( - self.boot_disk.id == block_device.id and - arch == "ppc64el") + return arch == "ppc64el" and block_device.id in self.grub_device_ids def _requires_bios_grub_partition(self, block_device): """Return True if block device requires the bios_grub partition.""" arch, _ = self.node.split_arch() bios_boot_method = self.node.get_bios_boot_method() - return ( - arch == "amd64" and - bios_boot_method != "uefi" and - block_device.size >= GPT_REQUIRED_SIZE) + return arch == "amd64" and bios_boot_method != "uefi" def _add_partition_operations(self): """Add all the partition operations. @@ -145,10 +144,14 @@ partitions = list(partition_table.partitions.order_by('id')) for idx, partition in enumerate(partitions): # If this is the first partition and prep or bios_grub - # partition is required then set boot_disk_first_partition - # so partition creation can occur in the correct order. - if (requires_prep or requires_bios_grub) and idx == 0: - self.boot_disk_first_partition = partition + # partition is required then track this as a first + # partition for boot + is_boot_partition = ( + (requires_prep or requires_bios_grub) and + block_device.id in self.grub_device_ids and + idx == 0) + if is_boot_partition: + self.boot_first_partitions.append(partition) self.operations["partition"].append(partition) def _add_format_and_mount_operations(self): @@ -184,6 +187,25 @@ filesystem.filesystem_group_id is None and filesystem.cache_set is None) + def _find_grub_devices(self): + """Save which devices should have grub installed.""" + for raid in self.operations["raid"]: + partition_ids, block_devices_ids = zip( + *raid.filesystems.values_list('partition', 'block_device')) + partition_ids = set(partition_ids) + partition_ids.discard(None) + block_devices_ids = set(block_devices_ids) + block_devices_ids.discard(None) + devices = PhysicalBlockDevice.objects.filter( + Q(id__in=block_devices_ids) | + Q(partitiontable__partitions__in=partition_ids)) + devices = list(devices.values_list('id', flat=True)) + if self.boot_disk.id in devices: + self.grub_device_ids = devices + + if not self.grub_device_ids: + self.grub_device_ids = [self.boot_disk.id] + def _generate_disk_operations(self): """Generate all disk operations.""" for block_device in self.operations["disk"]: @@ -218,30 +240,33 @@ add_prep_partition = False add_bios_grub_partition = False partition_table = block_device.get_partitiontable() + bios_boot_method = self.node.get_bios_boot_method() + node_arch, _ = self.node.split_arch() + should_install_grub = block_device.id in self.grub_device_ids + if partition_table is not None: disk_operation["ptable"] = self._get_ptable_type( partition_table) - elif block_device.id == self.boot_disk.id: - bios_boot_method = self.node.get_bios_boot_method() - node_arch, _ = self.node.split_arch() - if bios_boot_method in [ - "uefi", "powernv", "powerkvm"]: - disk_operation["ptable"] = "gpt" - if node_arch == "ppc64el": - add_prep_partition = True - elif (block_device.size >= GPT_REQUIRED_SIZE and - node_arch == "amd64"): - disk_operation["ptable"] = "gpt" - add_bios_grub_partition = True - else: - disk_operation["ptable"] = "msdos" + elif should_install_grub: + gpt_table = ( + bios_boot_method in ["uefi", "powernv", "powerkvm"] or + (bios_boot_method != "uefi" and node_arch == "amd64")) + disk_operation["ptable"] = "gpt" if gpt_table else "msdos" + add_prep_partition = ( + node_arch == "ppc64el" and + bios_boot_method in ("uefi", "powernv", "powerkvm")) + + # always add a boot partition for GPT without UEFI + add_bios_grub_partition = ( + disk_operation.get("ptable") == "gpt" and + node_arch == "amd64" and bios_boot_method != "uefi") # Set this disk to be the grub device if it's the boot disk and doesn't # require a prep partition. When a prep partition is required grub # must be installed on that partition and not in the partition header # of that disk. requires_prep = self._requires_prep_partition(block_device) - if self.boot_disk.id == block_device.id and not requires_prep: + if should_install_grub and not requires_prep: disk_operation["grub_device"] = True self.storage_config.append(disk_operation) @@ -301,20 +326,13 @@ def _generate_partition_operations(self): """Generate all partition operations.""" for partition in self.operations["partition"]: - if partition == self.boot_disk_first_partition: + if partition in self.boot_first_partitions: # This is the first partition in the boot disk and add prep # partition at the beginning of the partition table. device_name = partition.partition_table.block_device.get_name() if self._requires_prep_partition( partition.partition_table.block_device): self._generate_prep_partition(device_name) - elif self._requires_bios_grub_partition( - partition.partition_table.block_device): - self._generate_bios_grub_partition(device_name) - else: - raise ValueError( - "boot_disk_first_partition set when prep and " - "bios_grub partition are not required.") self._generate_partition_operation( partition, include_initial=False) else: @@ -456,12 +474,11 @@ } for filesystem in filesystem_group.filesystems.all(): block_or_partition = filesystem.get_parent() + name = block_or_partition.get_name() if filesystem.fstype == FILESYSTEM_TYPE.RAID: - raid_operation["devices"].append( - block_or_partition.get_name()) + raid_operation["devices"].append(name) elif filesystem.fstype == FILESYSTEM_TYPE.RAID_SPARE: - raid_operation["spare_devices"].append( - block_or_partition.get_name()) + raid_operation["spare_devices"].append(name) raid_operation["devices"] = sorted(raid_operation["devices"]) raid_operation["spare_devices"] = sorted( raid_operation["spare_devices"]) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/proxyconfig.py maas-2.3.5-6511-gf466fdb/src/maasserver/proxyconfig.py --- maas-2.3.0-6434-gd354690/src/maasserver/proxyconfig.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/proxyconfig.py 2018-08-17 02:41:34.000000000 +0000 @@ -71,6 +71,7 @@ cidrs = [subnet.cidr for subnet in allowed_subnets] http_proxy = Config.objects.get_config("http_proxy") + maas_proxy_port = Config.objects.get_config("maas_proxy_port") upstream_proxy_enabled = ( Config.objects.get_config("use_peer_proxy") and http_proxy) context = { @@ -83,6 +84,7 @@ 'snap_data_path': snappy.get_snap_data_path(), 'snap_common_path': snappy.get_snap_common_path(), 'upstream_peer_proxy': upstream_proxy_enabled, + 'maas_proxy_port': maas_proxy_port, } proxy_enabled = Config.objects.get_config("enable_http_proxy") diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/rpc/boot.py maas-2.3.5-6511-gf466fdb/src/maasserver/rpc/boot.py --- maas-2.3.0-6434-gd354690/src/maasserver/rpc/boot.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/rpc/boot.py 2018-08-17 02:41:34.000000000 +0000 @@ -243,21 +243,28 @@ purpose = machine.get_boot_purpose() # Log the request into the event log for that machine. - if (machine.status == NODE_STATUS.ENTERING_RESCUE_MODE and - purpose == 'commissioning'): + if (machine.status in [ + NODE_STATUS.ENTERING_RESCUE_MODE, + NODE_STATUS.RESCUE_MODE] and purpose == 'commissioning'): event_log_pxe_request(machine, 'rescue') else: event_log_pxe_request(machine, purpose) # Get the correct operating system and series based on the purpose # of the booting machine. + precise = False if purpose == "commissioning": osystem = Config.objects.get_config('commissioning_osystem') series = Config.objects.get_config('commissioning_distro_series') else: osystem = machine.get_osystem() series = machine.get_distro_series() - if purpose == "xinstall" and osystem != "ubuntu": + # XXX: roaksoax LP: #1739761 - Since the switch to squashfs (and + # drop of iscsi), precise is no longer deployable. To address a + # squashfs image is made available allowing it to be deployed in + # the commissioning ephemeral environment. + precise = True if series == "precise" else False + if purpose == "xinstall" and (osystem != "ubuntu" or precise): # Use only the commissioning osystem and series, for operating # systems other than Ubuntu. As Ubuntu supports HWE kernels, # and needs to use that kernel to perform the installation. @@ -273,7 +280,12 @@ # subarchitecture. Since Ubuntu does not support architecture specific # hardware enablement kernels(i.e a highbank hwe-t kernel on precise) # we give precedence to any kernel defined in the subarchitecture field - if subarch == "generic" and machine.hwe_kernel: + + # XXX: roaksoax LP: #1739761 - Do not override the subarch (used for + # the deployment ephemeral env) when deploying precise, provided that + # it uses the commissioning distro_series and hwe kernels are not + # needed. + if subarch == "generic" and machine.hwe_kernel and not precise: subarch = machine.hwe_kernel elif(subarch == "generic" and purpose == "commissioning" and diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/rpc/tests/test_boot.py maas-2.3.5-6511-gf466fdb/src/maasserver/rpc/tests/test_boot.py --- maas-2.3.0-6434-gd354690/src/maasserver/rpc/tests/test_boot.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/rpc/tests/test_boot.py 2018-08-17 02:41:34.000000000 +0000 @@ -429,6 +429,20 @@ self.assertThat( event_log_pxe_request, MockCalledOnceWith(node, 'rescue')) + def test__uses_rescue_mode_reboot_purpose(self): + # Regression test for LP:1749210 + rack_controller = factory.make_RackController() + local_ip = factory.make_ip_address() + remote_ip = factory.make_ip_address() + node = self.make_node(status=NODE_STATUS.RESCUE_MODE) + mac = node.get_boot_interface().mac_address + event_log_pxe_request = self.patch_autospec( + boot_module, 'event_log_pxe_request') + get_config( + rack_controller.system_id, local_ip, remote_ip, mac=mac) + self.assertThat( + event_log_pxe_request, MockCalledOnceWith(node, 'rescue')) + def test__calls_event_log_pxe_request(self): rack_controller = factory.make_RackController() local_ip = factory.make_ip_address() @@ -695,6 +709,27 @@ rack_controller.system_id, local_ip, remote_ip, mac=mac) self.assertEqual(distro_series, observed_config["release"]) + # XXX: roaksoax LP: #1739761 - Deploying precise is now done using + # the commissioning ephemeral environment. + def test__returns_commissioning_os_series_for_precise_xinstall(self): + self.patch(boot_module, 'get_boot_filenames').return_value = ( + None, None, None) + commissioning_series = "xenial" + Config.objects.set_config( + "commissioning_distro_series", commissioning_series) + distro_series = "precise" + rack_controller = factory.make_RackController() + local_ip = factory.make_ip_address() + remote_ip = factory.make_ip_address() + node = self.make_node( + status=NODE_STATUS.DEPLOYING, osystem='ubuntu', + distro_series=distro_series, primary_rack=rack_controller) + mac = node.get_boot_interface().mac_address + observed_config = get_config( + rack_controller.system_id, local_ip, remote_ip, mac=mac) + self.assertEqual(observed_config["release"], commissioning_series) + self.assertEqual(node.distro_series, distro_series) + def test__returns_commissioning_os_when_erasing_disks(self): self.patch(boot_module, 'get_boot_filenames').return_value = ( None, None, None) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/server_address.py maas-2.3.5-6511-gf466fdb/src/maasserver/server_address.py --- maas-2.3.0-6434-gd354690/src/maasserver/server_address.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/server_address.py 2018-08-17 02:41:34.000000000 +0000 @@ -110,17 +110,21 @@ raise UnresolvableHost("No address found for host %s." % hostname) if not link_local: addresses = [ip for ip in addresses if not ip.is_link_local()] + # Addresses must be returned in a consistent order, with the set of IP + # addresses found from get_maas_facing_server_host() (i.e. maas_url) + # coming first. + addresses = sorted(addresses) if include_alternates: maas_id = get_maas_id() if maas_id is not None: # Circular imports from maasserver.models import Subnet from maasserver.models import StaticIPAddress - # Keep track of the regions already represented. + # Don't include more than one alternate IP address, per region, + # per address-family. regions = set() alternate_ips = [] for ip in addresses: - regions.add(maas_id + str(ip.version)) if not ip.is_link_local(): # Since we only know that the IP address given in the MAAS # URL is reachable, alternates must be pulled from the same @@ -149,5 +153,10 @@ else: regions.add(id_plus_family) alternate_ips.append(ipa) - addresses.extend(alternate_ips) + # Append non-duplicate region IP addresses to the list of + # addresses to return. We don't want duplicates, but we + # also need to preserve the existing order. + for address in alternate_ips: + if address not in addresses: + addresses.append(address) return addresses Binary files /tmp/tmpNeRntK/HOzLjHA_6m/maas-2.3.0-6434-gd354690/src/maasserver/static/assets/images/icons/maas-favicon-32px.png and /tmp/tmpNeRntK/2QY2kacaqR/maas-2.3.5-6511-gf466fdb/src/maasserver/static/assets/images/icons/maas-favicon-32px.png differ diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/node_details.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/node_details.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/node_details.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/node_details.js 2018-08-17 02:41:34.000000000 +0000 @@ -33,7 +33,8 @@ option: null, allOptions: null, availableOptions: [], - error: null + error: null, + showing_confirmation: false }; $scope.power_types = GeneralManager.getData("power_types"); $scope.osinfo = GeneralManager.getData("osinfo"); @@ -600,12 +601,14 @@ $scope.action.optionChanged = function() { // Clear the action error. $scope.action.error = null; + $scope.action.showing_confirmation = false; }; // Cancel the action. $scope.actionCancel = function() { $scope.action.option = null; $scope.action.error = null; + $scope.action.showing_confirmation = false; }; // Perform the action. @@ -655,6 +658,11 @@ extra.testing_scripts.push('none'); } } else if($scope.action.option.name === "test") { + if($scope.node.status_code === 6 && + !$scope.action.showing_confirmation) { + $scope.action.showing_confirmation = true; + return; + } // Set the test options. extra.enable_ssh = $scope.commissionOptions.enableSSH; extra.testing_scripts = []; @@ -680,6 +688,7 @@ } $scope.action.option = null; $scope.action.error = null; + $scope.action.showing_confirmation = false; $scope.osSelection.$reset(); $scope.commissionOptions.enableSSH = false; $scope.commissionOptions.skipNetworking = false; @@ -1113,6 +1122,11 @@ // Only show a warning that tests have failed if there are failed tests // and the node isn't currently commissioning or testing. $scope.showFailedTestWarning = function() { + // Devices can't have failed tests and don't have status_code + // defined. + if($scope.node.node_type === 1 || !$scope.node.status_code) { + return false; + } switch($scope.node.status_code) { // NEW case 0: diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/node_details_storage.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/node_details_storage.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/node_details_storage.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/node_details_storage.js 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015-2017 Canonical Ltd. This software is licensed under the +/* Copyright 2015-2018 Canonical Ltd. This software is licensed under the * GNU Affero General Public License version 3 (see the file LICENSE). * * MAAS Node Storage Controller @@ -824,7 +824,7 @@ return available; }; - // Update the currect mode for the available section and the all + // Update the current mode for the available section and the all // selected value. $scope.updateAvailableSelection = function(force) { if(angular.isUndefined(force)) { @@ -1096,8 +1096,15 @@ } // Save the options. - MachinesManager.updateDisk( - $scope.node, disk.block_id, params); + if(disk.type === "partition") { + MachinesManager.updateFilesystem( + $scope.node, disk.block_id, disk.partition_id, + params.fstype, params.mount_point, + params.mount_options, params.tags); + } else { + MachinesManager.updateDisk( + $scope.node, disk.block_id, params); + } // Set the options on the object so no flicker occurs while waiting // for the new object to be received. @@ -1107,7 +1114,7 @@ disk.tags = disk.$options.tags; disk.$options = {}; - // If the mount_point is set the we need to transition this to + // If the mount_point is set then we need to transition this to // the filesystem section. if(angular.isString(disk.mount_point) && disk.mount_point !== "") { $scope.filesystems.push({ diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/nodes_list.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/nodes_list.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/nodes_list.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/nodes_list.js 2018-08-17 02:41:34.000000000 +0000 @@ -57,7 +57,9 @@ $scope.tabs.nodes.actionProgress = { total: 0, completed: 0, - errors: {} + errors: {}, + showing_confirmation: false, + affected_nodes: 0 }; $scope.tabs.nodes.osSelection = { osystem: null, @@ -96,7 +98,9 @@ $scope.tabs.devices.actionProgress = { total: 0, completed: 0, - errors: {} + errors: {}, + showing_confirmation: false, + affected_nodes: 0 }; $scope.tabs.devices.zoneSelection = null; @@ -124,7 +128,9 @@ $scope.tabs.controllers.actionProgress = { total: 0, completed: 0, - errors: {} + errors: {}, + showing_confirmation: false, + affected_nodes: 0 }; $scope.tabs.controllers.zoneSelection = null; $scope.tabs.controllers.syncStatuses = {}; @@ -154,7 +160,9 @@ $scope.tabs.switches.actionProgress = { total: 0, completed: 0, - errors: {} + errors: {}, + showing_confirmation: false, + affected_nodes: 0 }; $scope.tabs.switches.osSelection = { osystem: null, @@ -303,6 +311,8 @@ var progress = $scope.tabs[tab].actionProgress; progress.completed = progress.total = 0; progress.errors = {}; + progress.showing_confirmation = false; + progress.affected_nodes = 0; } // Add error to action progress and group error messages by nodes. @@ -587,6 +597,19 @@ extra.testing_scripts.push('none'); } } else if($scope.tabs[tab].actionOption.name === "test") { + if(!$scope.tabs[tab].actionProgress.showing_confirmation) { + var progress = $scope.tabs[tab].actionProgress; + for(i=0;i<$scope.tabs[tab].selectedItems.length;i++) { + if($scope.tabs[tab].selectedItems[i].status_code === 6) + { + progress.showing_confirmation = true; + progress.affected_nodes++; + } + } + if($scope.tabs[tab].actionProgress.affected_nodes != 0) { + return; + } + } // Set the test options. extra.enable_ssh = ( $scope.tabs[tab].commissionOptions.enableSSH); diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/tests/test_node_details.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/tests/test_node_details.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/tests/test_node_details.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/tests/test_node_details.js 2018-08-17 02:41:34.000000000 +0000 @@ -194,6 +194,7 @@ expect($scope.action.allOptions).toBeNull(); expect($scope.action.availableOptions).toEqual([]); expect($scope.action.error).toBeNull(); + expect($scope.action.showing_confirmation).toBe(false); expect($scope.osinfo).toBe(GeneralManager.getData("osinfo")); expect($scope.power_types).toBe(GeneralManager.getData("power_types")); expect($scope.osSelection.osystem).toBeNull(); @@ -995,6 +996,13 @@ $scope.actionCancel(); expect($scope.action.error).toBeNull(); }); + + it("resets showing_confirmation", function() { + var controller = makeController(); + $scope.action.showing_confirmation = true; + $scope.actionCancel(); + expect($scope.action.showing_confirmation).toBe(false); + }); }); describe("actionGo", function() { @@ -1136,6 +1144,20 @@ }); }); + it("sets showing_confirmation with testOptions", function() { + var controller = makeController(); + spyOn(MachinesManager, "performAction").and.returnValue( + $q.defer().promise); + node.status_code = 6; + $scope.node = node; + $scope.action.option = { + name: "test" + }; + $scope.actionGo(); + expect($scope.action.showing_confirmation).toBe(true); + expect(MachinesManager.performAction).not.toHaveBeenCalled(); + }); + it("calls performAction with releaseOptions", function() { var controller = makeController(); spyOn(MachinesManager, "performAction").and.returnValue( @@ -2275,6 +2297,14 @@ describe("showFailedTestWarning", function() { + it("returns false when device", function() { + var controller = makeController(); + $scope.node = { + node_type: 1 + }; + expect($scope.showFailedTestWarning()).toBe(false); + }); + it("returns false when new, commissioning, or testing", function() { var controller = makeController(); $scope.node = node; diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/tests/test_node_details_storage.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/tests/test_node_details_storage.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/tests/test_node_details_storage.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/tests/test_node_details_storage.js 2018-08-17 02:41:34.000000000 +0000 @@ -3066,8 +3066,9 @@ it("calls updateDisk with new name for logical volume", function() { var controller = makeController(); + var name = "vg0-lvnew"; var disk = { - name: "vg0-lvnew", + name: name, type: "virtual", parent_type: "lvm-vg", block_id: makeInteger(0, 100), @@ -3084,9 +3085,30 @@ spyOn(MachinesManager, "updateDisk"); $scope.availableConfirmEdit(disk); + expect(disk.name).toBe(name); expect(MachinesManager.updateDisk).toHaveBeenCalled(); }); + it("calls updateFilesystem for partition", function() { + var controller = makeController(); + var name = makeName("name"); + var disk = { + name: "", + type: "partition", + $options: { + mountPoint: "" + }, + original: { + name: name + } + }; + spyOn(MachinesManager, "updateFilesystem"); + + $scope.availableConfirmEdit(disk); + expect(disk.name).toBe(name); + expect(MachinesManager.updateFilesystem).toHaveBeenCalled(); + }); + }); describe("canCreateBcache", function() { diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/tests/test_nodes_list.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/tests/test_nodes_list.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/controllers/tests/test_nodes_list.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/controllers/tests/test_nodes_list.js 2018-08-17 02:41:34.000000000 +0000 @@ -1180,10 +1180,18 @@ makeInteger(0, 10); $scope.tabs[tab].actionProgress.errors[makeName("error")] = [{}]; + $scope.tabs[tab].actionProgress.showing_confirmation = + true; + $scope.tabs[tab].actionProgress.affected_nodes = + makeInteger(0, 10); $scope.actionCancel(tab); expect($scope.tabs[tab].actionProgress.total).toBe(0); expect($scope.tabs[tab].actionProgress.completed).toBe(0); expect($scope.tabs[tab].actionProgress.errors).toEqual({}); + expect($scope.tabs[ + tab].actionProgress.showing_confirmation).toBe(false); + expect($scope.tabs[ + tab].actionProgress.affected_nodes).toBe(0); }); }); @@ -1560,6 +1568,26 @@ }); }); + it("sets showing_confirmation with testOptions", + function() { + var controller = makeController(); + var object = makeObject("nodes"); + object.status_code = 6; + var spy = spyOn( + $scope.tabs.nodes.manager, + "performAction").and.returnValue( + $q.defer().promise); + $scope.tabs.nodes.actionOption = { name: "test" }; + $scope.tabs.nodes.selectedItems = [object]; + $scope.actionGo("nodes"); + expect($scope.tabs[ + "nodes"].actionProgress.showing_confirmation).toBe( + true); + expect($scope.tabs[ + "nodes"].actionProgress.affected_nodes).toBe(1); + expect(spy).not.toHaveBeenCalled(); + }); + it("calls performAction with releaseOptions", function() { var controller = makeController(); diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/directives/script_runtime.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/directives/script_runtime.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/directives/script_runtime.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/directives/script_runtime.js 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2017 Canonical Ltd. This software is licensed under the +/* Copyright 2017-2018 Canonical Ltd. This software is licensed under the * GNU Affero General Public License version 3 (see the file LICENSE). * * Script runtime counter directive. @@ -36,9 +36,8 @@ function incrementCounter() { if(($scope.scriptStatus === 1 || $scope.scriptStatus === 7) && $scope.startTime) { - var start_date = new Date(null); - start_date.setSeconds($scope.startTime); - var seconds = Math.floor((Date.now() - start_date) / 1000); + var seconds = Math.floor( + (Date.now() / 1000) - $scope.startTime); var minutes = Math.floor(seconds / 60); var hours = Math.floor(minutes / 60); var days = Math.floor(hours / 24); diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/directives/tests/test_script_runtime.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/directives/tests/test_script_runtime.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/directives/tests/test_script_runtime.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/directives/tests/test_script_runtime.js 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2017 Canonical Ltd. This software is licensed under the +/* Copyright 2017-2018 Canonical Ltd. This software is licensed under the * GNU Affero General Public License version 3 (see the file LICENSE). * * Unit tests for script runtime directive. @@ -101,4 +101,17 @@ expect(spanElement.text()).toEqual( '2 days, 0:00:00 of ~' + estimatedRunTime); }); + + it('regression test for LP:1757153', function() { + var startTime = (Date.now() / 1000) - 1; + var estimatedRunTime = '0:00:54'; + var scriptStatus = 1; + var directive = compileDirective( + startTime, null, estimatedRunTime, scriptStatus); + // Flush should not cause the passed time to change. + var spanElement = directive.find('span'); + expect(spanElement).toBeDefined(); + expect(spanElement.text()).toEqual( + '0:00:01 of ~' + estimatedRunTime); + }); }); diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/factories/node_results.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/factories/node_results.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/factories/node_results.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/factories/node_results.js 2018-08-17 02:41:34.000000000 +0000 @@ -136,18 +136,20 @@ result.showing_results = false; result.showing_history = false; result.showing_menu = false; + result.result_section = "scripts"; if(result.result_type === 0) { results = this.commissioning_results; - }else if(result.result_type === 1) { + } else if(result.result_type === 1) { // Installation results are not split into hardware types or // have subtext labels. this._addOrReplace(this.installation_results, result); return; - }else{ - // Store all remaining result types as test results incase + } else { + // Store all remaining result types as test results in case // another result type is ever added. results = this.testing_results; + result.result_section = "tests"; } var i; // Fallback to storing results in other results incase a new type @@ -179,7 +181,7 @@ break; } } - }else{ + } else { // Other hardware types are not split into individual // components. if(!angular.isArray(hardware_type_results[null])) { diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/factories/tests/test_node_results.js maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/factories/tests/test_node_results.js --- maas-2.3.0-6434-gd354690/src/maasserver/static/js/angular/factories/tests/test_node_results.js 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/js/angular/factories/tests/test_node_results.js 2018-08-17 02:41:34.000000000 +0000 @@ -154,11 +154,16 @@ } } results.push(old_result); + var result_section = "tests"; + if (result_type_name === "commissioning") { + result_section = "scripts"; + } var result = { name: old_result.name, status: makeInteger(0, 100), status_name: makeName("status_name"), result_type: parseInt(result_type, 10), + result_section: result_section, hardware_type: parseInt(hardware_type, 10), physical_blockdevice: null, showing_results: false, @@ -172,6 +177,7 @@ status: result.status, status_name: result.status_name, result_type: parseInt(result_type, 10), + result_section: result_section, hardware_type: parseInt(hardware_type, 10), physical_blockdevice: null, showing_results: old_result.showing_results, @@ -192,11 +198,14 @@ }]; var manager = NodeResultsManagerFactory.getManager( node, result_type_name); + var result_section = ( + result_type_name === "commissioning") ? "scripts" : "tests"; var result = { name: makeName("name"), status: makeInteger(0, 100), status_name: makeName("status_name"), result_type: parseInt(result_type, 10), + result_section: result_section, hardware_type: 3, physical_blockdevice: node.disks[0].id, showing_results: false, @@ -254,6 +263,8 @@ } } results.push(old_result); + var result_section = ( + result_type_name === "commissioning") ? "scripts" : "tests"; var result = { name: old_result.name, status: makeInteger(0, 100), @@ -272,6 +283,7 @@ status: result.status, status_name: result.status_name, result_type: parseInt(result_type, 10), + result_section: result_section, hardware_type: 3, physical_blockdevice: node.disks[0].id, showing_results: old_result.showing_results, @@ -325,6 +337,7 @@ status: makeInteger(0, 100), status_name: makeName("status_name"), result_type: 1, + result_section: "scripts", hardware_type: 0, physical_blockdevice: null, showing_results: false, @@ -338,6 +351,7 @@ status: result.status, status_name: result.status_name, result_type: 1, + result_section: "scripts", hardware_type: 0, physical_blockdevice: null, showing_results: old_result.showing_results, diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/partials/ipranges.html maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/ipranges.html --- maas-2.3.0-6434-gd354690/src/maasserver/static/partials/ipranges.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/ipranges.html 2018-08-17 02:41:34.000000000 +0000 @@ -14,7 +14,7 @@ data-ng-class="{ 'is-active': isIPRangeInEditMode(iprange) || isIPRangeInDeleteMode(iprange)}">
{$ iprange.start_ip $}
{$ iprange.end_ip $}
-
{$ iprange.type == "dynamic" ? "MAAS" : iprange.user_username $}
+
{$ iprange.type == "dynamic" ? "MAAS" : iprange.user $}
{$ iprange.type == "dynamic" ? "Dynamic" : "Reserved" $}
{$ iprange.type == "dynamic" ? "Dynamic" : iprange.comment $}
diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/partials/node-details.html maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/node-details.html --- maas-2.3.0-6434-gd354690/src/maasserver/static/partials/node-details.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/node-details.html 2018-08-17 02:41:34.000000000 +0000 @@ -137,7 +137,15 @@
- +
+

+ Node is currently deployed. Are you sure you want to continue to test hardware? +

+
+ + +
+

diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/partials/nodes-list.html maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/nodes-list.html --- maas-2.3.0-6434-gd354690/src/maasserver/static/partials/nodes-list.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/nodes-list.html 2018-08-17 02:41:34.000000000 +0000 @@ -220,7 +220,15 @@ cannot be {$ tabs[tab].actionOption.sentence $}, because an SSH key has not been added to your account. To add an SSH key, visit your account page.

- +
+

+ {$ tabs[tab].actionProgress.affected_nodes $} of {$ tabs[tab].selectedItems.length $} are in a deployed state. Are you sure you want to continue to test hardware? +

+
+ + +
+

diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/partials/script-results-list.html maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/script-results-list.html --- maas-2.3.0-6434-gd354690/src/maasserver/static/partials/script-results-list.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/script-results-list.html 2018-08-17 02:41:34.000000000 +0000 @@ -44,8 +44,8 @@

@@ -81,7 +81,7 @@

- +

diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/partials/subnet-details.html maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/subnet-details.html --- maas-2.3.0-6434-gd354690/src/maasserver/static/partials/subnet-details.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/subnet-details.html 2018-08-17 02:41:34.000000000 +0000 @@ -142,7 +142,7 @@ options="fabric.id as fabric.name for fabric in fabrics | orderBy:'name'" label-width="two" input-width="three">
Space
diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/static/partials/vlan-details.html maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/vlan-details.html --- maas-2.3.0-6434-gd354690/src/maasserver/static/partials/vlan-details.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/static/partials/vlan-details.html 2018-08-17 02:41:34.000000000 +0000 @@ -220,7 +220,7 @@
-
+

VLAN Summary

@@ -268,12 +268,10 @@
-
+
-
Fabric
-
- {$ vlanDetails.fabric.name $} -
+
Rack controllers
@@ -283,8 +281,11 @@
+
+
+
-
+
diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/stats.py maas-2.3.5-6511-gf466fdb/src/maasserver/stats.py --- maas-2.3.0-6434-gd354690/src/maasserver/stats.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/stats.py 2018-08-17 02:41:34.000000000 +0000 @@ -10,6 +10,7 @@ from datetime import timedelta +from django.db.models import Sum from maasserver.models import Config from maasserver.utils.orm import transactional from maasserver.utils.threads import deferToDatabase @@ -29,9 +30,20 @@ import requests +def get_machine_stats(): + nodes = Node.objects.all() + machines = nodes.filter(node_type=NODE_TYPE.MACHINE) + # Rather overall amount of stats for machines. + return machines.aggregate( + total_cpu=Sum('cpu_count'), total_mem=Sum('memory'), + total_storage=Sum('blockdevice__size')) + + def get_maas_stats(): + # Get all node types to get count values node_types = Node.objects.values_list('node_type', flat=True) node_types = Counter(node_types) + stats = get_machine_stats() return json.dumps({ "controllers": { @@ -43,7 +55,8 @@ "nodes": { "machines": node_types.get(NODE_TYPE.MACHINE, 0), "devices": node_types.get(NODE_TYPE.DEVICE, 0), - } + }, + "machine_stats": stats, }) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/templates/maasserver/base.html maas-2.3.5-6511-gf466fdb/src/maasserver/templates/maasserver/base.html --- maas-2.3.0-6434-gd354690/src/maasserver/templates/maasserver/base.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/templates/maasserver/base.html 2018-08-17 02:41:34.000000000 +0000 @@ -14,7 +14,7 @@ {% block title %}{% endblock %} | {% include "maasserver/site_title.html" %} - + {% block css-conf %} diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/templates/maasserver/index.html maas-2.3.5-6511-gf466fdb/src/maasserver/templates/maasserver/index.html --- maas-2.3.0-6434-gd354690/src/maasserver/templates/maasserver/index.html 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/templates/maasserver/index.html 2018-08-17 02:41:34.000000000 +0000 @@ -14,7 +14,7 @@ - + {% include "maasserver/css-conf.html" %} diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/testing/factory.py maas-2.3.5-6511-gf466fdb/src/maasserver/testing/factory.py --- maas-2.3.0-6434-gd354690/src/maasserver/testing/factory.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/testing/factory.py 2018-08-17 02:41:34.000000000 +0000 @@ -1324,16 +1324,26 @@ def make_IPRange( self, subnet=None, start_ip=None, end_ip=None, comment=None, - user=None, type=IPRANGE_TYPE.DYNAMIC): + user=None, alloc_type=None): + if alloc_type is None: + alloc_type = ( + IPRANGE_TYPE.RESERVED if user else IPRANGE_TYPE.DYNAMIC) + if subnet is None and start_ip is None and end_ip is None: subnet = self.make_ipv4_Subnet_with_IPRanges() - return subnet.get_dynamic_ranges().first() + iprange = subnet.get_dynamic_ranges().first() + iprange.comment = comment + iprange.user = user + iprange.type = alloc_type + iprange.save() + return iprange + # If any of these values are provided, they must all be provided. assert subnet is not None assert start_ip is not None assert end_ip is not None iprange = IPRange( - subnet=subnet, start_ip=start_ip, end_ip=end_ip, type=type, + subnet=subnet, start_ip=start_ip, end_ip=end_ip, type=alloc_type, comment=comment, user=user) iprange.save() return iprange @@ -1372,13 +1382,13 @@ subnet.vlan.dhcp_on = True subnet.vlan.save() self.make_IPRange( - subnet, type=IPRANGE_TYPE.DYNAMIC, + subnet, alloc_type=IPRANGE_TYPE.DYNAMIC, start_ip=str(IPAddress(network.first + 2)), end_ip=str(IPAddress(network.first + range_size + 2))) # Create a "static range" for this Subnet. if not with_static_range: self.make_IPRange( - subnet, type=IPRANGE_TYPE.RESERVED, + subnet, alloc_type=IPRANGE_TYPE.RESERVED, start_ip=str(IPAddress(network.last - range_size - 2)), end_ip=str(IPAddress(network.last - 2))) return reload_object(subnet) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_bootresources.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_bootresources.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_bootresources.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_bootresources.py 2018-08-17 02:41:34.000000000 +0000 @@ -83,7 +83,10 @@ MAASServerTestCase, MAASTransactionServerTestCase, ) -from maasserver.utils import absolute_reverse +from maasserver.utils import ( + absolute_reverse, + get_maas_user_agent, +) from maasserver.utils.django_urls import reverse from maasserver.utils.orm import ( get_one, @@ -114,7 +117,6 @@ asynchronous, DeferredValue, ) -from provisioningserver.utils.version import get_maas_version_user_agent from testtools.matchers import ( Contains, ContainsAll, @@ -1561,7 +1563,7 @@ self.assertThat( mock_UrlMirrorReader, MockCalledOnceWith( - ANY, policy=ANY, user_agent=get_maas_version_user_agent())) + ANY, policy=ANY, user_agent=get_maas_user_agent())) def test_download_boot_resources_fallsback_to_no_user_agent(self): self.patch(bootresources.BootResourceRepoWriter, 'sync') @@ -1579,7 +1581,7 @@ MockCallsMatch( call( ANY, policy=ANY, - user_agent=get_maas_version_user_agent()), + user_agent=get_maas_user_agent()), call( ANY, policy=ANY))) @@ -1735,7 +1737,7 @@ self.expectThat( image_descriptions, MockCalledOnceWith( - [sentinel.source], get_maas_version_user_agent())) + [sentinel.source], get_maas_user_agent())) self.expectThat( map_products, MockCalledOnceWith(descriptions)) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_bootsources.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_bootsources.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_bootsources.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_bootsources.py 2018-08-17 02:41:34.000000000 +0000 @@ -42,6 +42,7 @@ MAASTransactionServerTestCase, ) from maasserver.tests.test_bootresources import SimplestreamsEnvFixture +from maasserver.utils import get_maas_user_agent from maastesting.matchers import MockCalledOnceWith from provisioningserver.config import DEFAULT_IMAGES_URL from provisioningserver.import_images import ( @@ -51,7 +52,6 @@ BootImageMapping, ) from provisioningserver.import_images.helpers import ImageSpec -from provisioningserver.utils.version import get_maas_version_user_agent from requests.exceptions import ConnectionError from testtools.matchers import HasLength @@ -258,6 +258,23 @@ capture.env['http_proxy'], capture.env['https_proxy'], capture.env['no_proxy'])) + def test__has_env_http_and_https_proxy_set_with_custom_no_proxy(self): + proxy_address = factory.make_name('proxy') + Config.objects.set_config('http_proxy', proxy_address) + Config.objects.set_config('boot_images_no_proxy', True) + capture = ( + patch_and_capture_env_for_download_all_image_descriptions(self)) + factory.make_BootSource( + keyring_data=b'1234', + url=b'http://192.168.1.100:8080/ephemeral-v3/') + cache_boot_sources() + no_proxy_hosts = '127.0.0.1,localhost,192.168.1.100' + self.assertEqual( + (proxy_address, proxy_address, no_proxy_hosts), + ( + capture.env['http_proxy'], capture.env['https_proxy'], + capture.env['no_proxy'])) + def test__passes_user_agent_with_maas_version(self): mock_download = self.patch( bootsources, 'download_all_image_descriptions') @@ -266,7 +283,7 @@ self.assertThat( mock_download, MockCalledOnceWith( - ANY, user_agent=get_maas_version_user_agent())) + ANY, user_agent=get_maas_user_agent())) @skip("XXX: GavinPanella 2015-12-04 bug=1546235: Fails spuriously.") def test__doesnt_have_env_http_and_https_proxy_set_if_disabled(self): diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_commands_edit_named_options.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_commands_edit_named_options.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_commands_edit_named_options.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_commands_edit_named_options.py 2018-08-17 02:41:34.000000000 +0000 @@ -120,14 +120,14 @@ self.assertContentFailsWithMessage( OPTIONS_FILE, "Failed to make a backup") - def test_does_not_remove_existing_forwarders_config(self): + def test_remove_existing_forwarders_config(self): options_file = self.make_file(contents=OPTIONS_FILE_WITH_FORWARDERS) call_command( "edit_named_options", config_path=options_file, stdout=self.stdout) options = read_isc_file(options_file) - self.assertThat(make_isc_string(options), Contains('forwarders')) + self.assertThat(make_isc_string(options), Not(Contains('forwarders'))) def test_removes_existing_forwarders_config_if_migrate_set(self): options_file = self.make_file(contents=OPTIONS_FILE_WITH_FORWARDERS) @@ -152,7 +152,7 @@ # that's now in the included file). options = read_isc_file(options_file) self.assertThat( - make_isc_string(options), Contains('dnssec-validation')) + make_isc_string(options), Not(Contains('dnssec-validation'))) def test_removes_existing_dnssec_validation_config_if_migration_set(self): options_file = self.make_file(contents=OPTIONS_FILE_WITH_DNSSEC) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_compose_preseed.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_compose_preseed.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_compose_preseed.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_compose_preseed.py 2018-08-17 02:41:34.000000000 +0000 @@ -50,6 +50,7 @@ ("ipv6", dict( default_region_ip=None, rack='2001:db8::1', + maas_proxy_port='', result='http://[2001:db8::1]:8000/', enable=True, use_peer_proxy=False, @@ -57,6 +58,7 @@ ("ipv4", dict( default_region_ip=None, rack='10.0.1.1', + maas_proxy_port=8000, result='http://10.0.1.1:8000/', enable=True, use_peer_proxy=False, @@ -64,6 +66,7 @@ ("builtin", dict( default_region_ip=None, rack='region.example.com', + maas_proxy_port=8000, result='http://region.example.com:8000/', enable=True, use_peer_proxy=False, @@ -71,6 +74,7 @@ ("external", dict( default_region_ip=None, rack='region.example.com', + maas_proxy_port='', result='http://proxy.example.com:111/', enable=True, use_peer_proxy=False, @@ -78,6 +82,7 @@ ("peer-proxy", dict( default_region_ip=None, rack='region.example.com', + maas_proxy_port='', result='http://region.example.com:8000/', enable=True, use_peer_proxy=True, @@ -85,6 +90,7 @@ ("disabled", dict( default_region_ip=None, rack='example.com', + maas_proxy_port=8000, result=None, enable=False, use_peer_proxy=False, @@ -95,6 +101,7 @@ ("ipv6_default", dict( default_region_ip='2001:db8::2', rack='', + maas_proxy_port=8000, result='http://[2001:db8::2]:8000/', enable=True, use_peer_proxy=False, @@ -102,6 +109,7 @@ ("ipv4_default", dict( default_region_ip='10.0.1.2', rack='', + maas_proxy_port=8000, result='http://10.0.1.2:8000/', enable=True, use_peer_proxy=False, @@ -109,6 +117,7 @@ ("builtin_default", dict( default_region_ip='region.example.com', rack='', + maas_proxy_port=8000, result='http://region.example.com:8000/', enable=True, use_peer_proxy=False, @@ -116,6 +125,7 @@ ("external_default", dict( default_region_ip='10.0.0.1', rack='', + maas_proxy_port=8000, result='http://proxy.example.com:111/', enable=True, use_peer_proxy=False, @@ -123,6 +133,7 @@ ("peer-proxy_default", dict( default_region_ip='region2.example.com', rack='', + maas_proxy_port=8000, result='http://region2.example.com:8000/', enable=True, use_peer_proxy=True, @@ -130,10 +141,19 @@ ("disabled_default", dict( default_region_ip='10.0.0.1', rack='', + maas_proxy_port=8000, result=None, enable=False, use_peer_proxy=False, http_proxy='')), + ("changed-maas_proxy_port", dict( + default_region_ip='region2.example.com', + rack='', + maas_proxy_port=9000, + result='http://region2.example.com:9000/', + enable=True, + use_peer_proxy=True, + http_proxy='http://proxy.example.com:111/')), ) def test__returns_correct_url(self): @@ -153,6 +173,9 @@ Config.objects.set_config("enable_http_proxy", self.enable) Config.objects.set_config("http_proxy", self.http_proxy) Config.objects.set_config("use_peer_proxy", self.use_peer_proxy) + if self.maas_proxy_port: + Config.objects.set_config( + "maas_proxy_port", self.maas_proxy_port) actual = get_apt_proxy( node.get_boot_rack_controller(), default_region_ip=self.default_region_ip) @@ -502,6 +525,46 @@ self.assertNotIn('apt_proxy', preseed) + # LP: #1743966 - Test for archive key work around + def test_compose_preseed_for_curtin_and_trusty_aptsources(self): + # Disable boot source cache signals. + self.addCleanup(bootsources.signals.enable) + bootsources.signals.disable() + + rack_controller = factory.make_RackController() + node = factory.make_Node( + interface=True, status=NODE_STATUS.READY, osystem='ubuntu', + distro_series='trusty') + nic = node.get_boot_interface() + nic.vlan.dhcp_on = True + nic.vlan.primary_rack = rack_controller + nic.vlan.save() + self.useFixture(RunningClusterRPCFixture()) + preseed = yaml.safe_load( + compose_preseed(PRESEED_TYPE.CURTIN, node)) + + self.assertIn('apt_sources', preseed) + + # LP: #1743966 - Test for archive key work around + def test_compose_preseed_for_curtin_xenial_not_aptsources(self): + # Disable boot source cache signals. + self.addCleanup(bootsources.signals.enable) + bootsources.signals.disable() + + rack_controller = factory.make_RackController() + node = factory.make_Node( + interface=True, status=NODE_STATUS.READY, osystem='ubuntu', + distro_series='xenial') + nic = node.get_boot_interface() + nic.vlan.dhcp_on = True + nic.vlan.primary_rack = rack_controller + nic.vlan.save() + self.useFixture(RunningClusterRPCFixture()) + preseed = yaml.safe_load( + compose_preseed(PRESEED_TYPE.CURTIN, node)) + + self.assertNotIn('apt_sources', preseed) + def test_compose_preseed_with_osystem_compose_preseed(self): os_name = factory.make_name('os') osystem = make_osystem(self, os_name, [BOOT_IMAGE_PURPOSE.XINSTALL]) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_preseed_network.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_preseed_network.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_preseed_network.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_preseed_network.py 2018-08-17 02:41:34.000000000 +0000 @@ -5,7 +5,6 @@ __all__ = [] -from operator import attrgetter import random from textwrap import dedent @@ -15,7 +14,6 @@ IPADDRESS_FAMILY, IPADDRESS_TYPE, ) -from maasserver.models.staticroute import StaticRoute from maasserver.preseed_network import ( compose_curtin_network_config, NodeNetworkConfiguration, @@ -228,31 +226,6 @@ ret += " - type: dhcp6\n" return ret - def collectRoutesConfig(self, node): - routes = set() - interfaces = node.interface_set.filter(enabled=True).order_by('name') - for iface in interfaces: - addresses = iface.ip_addresses.exclude( - alloc_type__in=[ - IPADDRESS_TYPE.DISCOVERED, - IPADDRESS_TYPE.DHCP, - ]).order_by('id') - for address in addresses: - subnet = address.subnet - if subnet is not None: - routes.update( - StaticRoute.objects.filter( - source=subnet).order_by('id')) - config = "" - for route in sorted(routes, key=attrgetter('id')): - config += self.ROUTE_CONFIG % { - 'id': route.id, - 'destination': route.destination.cidr, - 'gateway': route.gateway_ip, - 'metric': route.metric, - } - return config - def collectDNSConfig(self, node, ipv4=True, ipv6=True): config = "- type: nameserver\n address: %s\n search:\n" % ( repr(node.get_default_dns_servers(ipv4=ipv4, ipv6=ipv6))) @@ -466,29 +439,14 @@ self.assertNetworkConfig(net_config, config) -class TestNetworkLayoutWithRoutes( - MAASServerTestCase, AssertNetworkConfigMixin): - - def test__renders_expected_output(self): - node = factory.make_Node_with_Interface_on_Subnet( - interface_count=2) - for iface in node.interface_set.filter(enabled=True): - subnet = iface.vlan.subnet_set.first() - factory.make_StaticRoute(source=subnet) - factory.make_StaticIPAddress( - interface=iface, subnet=subnet) - net_config = self.collect_interface_config(node) - net_config += self.collectRoutesConfig(node) - net_config += self.collectDNSConfig(node) - config = compose_curtin_network_config(node) - self.assertNetworkConfig(net_config, config) - - class TestNetplan(MAASServerTestCase): def _render_netplan_dict(self, node): return NodeNetworkConfiguration(node, version=2).config + def _render_v1_dict(self, node): + return NodeNetworkConfiguration(node, version=1).config + def test__single_ethernet_interface(self): node = factory.make_Node() factory.make_Interface( @@ -569,7 +527,7 @@ } self.expectThat(netplan, Equals(expected_netplan)) - def test__bond_with_params(self): + def test__non_lacp_bond_with_params(self): node = factory.make_Node() eth0 = factory.make_Interface( node=node, name='eth0', mac_address="00:01:02:03:04:05") @@ -581,6 +539,7 @@ "bond_mode": "active-backup", "bond_lacp_rate": "slow", "bond_xmit_hash_policy": "layer2", + "bond_num_grat_arp": 3, }) netplan = self._render_netplan_dict(node) expected_netplan = { @@ -604,6 +563,51 @@ 'mtu': 1500, 'parameters': { "mode": "active-backup", + "transmit-hash-policy": "layer2", + "gratuitous-arp": 3, + }, + }, + }, + } + } + self.expectThat(netplan, Equals(expected_netplan)) + + def test__lacp_bond_with_params(self): + node = factory.make_Node() + eth0 = factory.make_Interface( + node=node, name='eth0', mac_address="00:01:02:03:04:05") + eth1 = factory.make_Interface( + node=node, name='eth1', mac_address="02:01:02:03:04:05") + factory.make_Interface( + INTERFACE_TYPE.BOND, + node=node, name='bond0', parents=[eth0, eth1], params={ + "bond_mode": "802.3ad", + "bond_lacp_rate": "slow", + "bond_xmit_hash_policy": "layer2", + "bond_num_grat_arp": 3, + }) + netplan = self._render_netplan_dict(node) + expected_netplan = { + 'network': { + 'version': 2, + 'ethernets': { + 'eth0': { + 'match': {'macaddress': '00:01:02:03:04:05'}, + 'mtu': 1500, + 'set-name': 'eth0' + }, + 'eth1': { + 'match': {'macaddress': '02:01:02:03:04:05'}, + 'mtu': 1500, + 'set-name': 'eth1' + }, + }, + 'bonds': { + 'bond0': { + 'interfaces': ['eth0', 'eth1'], + 'mtu': 1500, + 'parameters': { + "mode": "802.3ad", "lacp-rate": "slow", "transmit-hash-policy": "layer2", }, @@ -613,6 +617,51 @@ } self.expectThat(netplan, Equals(expected_netplan)) + def test__active_backup_with_legacy_parameter(self): + node = factory.make_Node() + eth0 = factory.make_Interface( + node=node, name='eth0', mac_address="00:01:02:03:04:05") + eth1 = factory.make_Interface( + node=node, name='eth1', mac_address="02:01:02:03:04:05") + factory.make_Interface( + INTERFACE_TYPE.BOND, + node=node, name='bond0', parents=[eth0, eth1], params={ + "bond_mode": "active-backup", + "bond_lacp_rate": "slow", + "bond_xmit_hash_policy": "layer2", + "bond_num_unsol_na": 3, + }) + netplan = self._render_netplan_dict(node) + expected_netplan = { + 'network': { + 'version': 2, + 'ethernets': { + 'eth0': { + 'match': {'macaddress': '00:01:02:03:04:05'}, + 'mtu': 1500, + 'set-name': 'eth0' + }, + 'eth1': { + 'match': {'macaddress': '02:01:02:03:04:05'}, + 'mtu': 1500, + 'set-name': 'eth1' + }, + }, + 'bonds': { + 'bond0': { + 'interfaces': ['eth0', 'eth1'], + 'mtu': 1500, + 'parameters': { + "mode": "active-backup", + "transmit-hash-policy": "layer2", + "gratuitous-arp": 3 + }, + }, + }, + } + } + self.expectThat(netplan, Equals(expected_netplan)) + def test__bridge(self): node = factory.make_Node() eth0 = factory.make_Interface( @@ -733,3 +782,208 @@ } } self.expectThat(netplan, Equals(expected_netplan)) + + def test__multiple_ethernet_interfaces_with_routes(self): + node = factory.make_Node() + vlan = factory.make_VLAN() + subnet = factory.make_Subnet( + cidr='10.0.0.0/24', gateway_ip='10.0.0.1', + dns_servers=[]) + subnet2 = factory.make_Subnet( + cidr='10.0.1.0/24', gateway_ip='10.0.1.1', + dns_servers=[]) + dest_subnet = factory.make_Subnet(cidr='192.168.0.0/24') + eth0 = factory.make_Interface( + node=node, name='eth0', mac_address="00:01:02:03:04:05", vlan=vlan) + eth1 = factory.make_Interface( + node=node, name='eth1', mac_address="02:01:02:03:04:05") + node.boot_interface = eth0 + node.save() + factory.make_StaticIPAddress( + interface=eth0, subnet=subnet, ip='10.0.0.4', + alloc_type=IPADDRESS_TYPE.STICKY) + factory.make_StaticIPAddress( + interface=eth1, subnet=subnet2, ip='10.0.1.4', + alloc_type=IPADDRESS_TYPE.STICKY) + factory.make_StaticRoute( + source=subnet, gateway_ip='10.0.0.3', destination=dest_subnet, + metric=42) + factory.make_StaticRoute( + source=subnet2, gateway_ip='10.0.1.3', destination=dest_subnet, + metric=43) + # Make sure we know when and where the default DNS server will be used. + get_default_dns_servers_mock = self.patch( + node, 'get_default_dns_servers') + get_default_dns_servers_mock.return_value = ['127.0.0.2'] + netplan = self._render_netplan_dict(node) + expected_netplan = { + 'network': { + 'version': 2, + 'ethernets': { + 'eth0': { + 'gateway': '10.0.0.1', + 'match': {'macaddress': '00:01:02:03:04:05'}, + 'mtu': 1500, + 'set-name': 'eth0', + 'addresses': ['10.0.0.4/24'], + 'routes': [{ + 'to': '192.168.0.0/24', + 'via': '10.0.0.3', + 'metric': 42, + }], + }, + 'eth1': { + 'match': {'macaddress': '02:01:02:03:04:05'}, + 'mtu': 1500, + 'set-name': 'eth1', + 'addresses': ['10.0.1.4/24'], + 'routes': [{ + 'to': '192.168.0.0/24', + 'via': '10.0.1.3', + 'metric': 43, + }], + }, + }, + }, + } + self.expectThat(netplan, Equals(expected_netplan)) + v1 = self._render_v1_dict(node) + expected_v1 = { + 'network': { + 'version': 1, + 'config': [ + { + 'id': 'eth0', + 'mac_address': '00:01:02:03:04:05', + 'mtu': 1500, + 'name': 'eth0', + 'subnets': [{ + 'address': '10.0.0.4/24', + 'gateway': '10.0.0.1', + 'type': 'static', + 'routes': [{ + 'destination': '192.168.0.0/24', + 'gateway': '10.0.0.3', + 'metric': 42 + }], + }], + 'type': 'physical' + }, + { + 'id': 'eth1', + 'mac_address': '02:01:02:03:04:05', + 'mtu': 1500, + 'name': 'eth1', + 'subnets': [{ + 'address': '10.0.1.4/24', + 'type': 'static', + 'routes': [{ + 'destination': '192.168.0.0/24', + 'gateway': '10.0.1.3', + 'metric': 43, + }], + }], + 'type': 'physical' + }, + { + 'address': ['127.0.0.2'], + 'search': ['maas'], + 'type': 'nameserver' + } + ], + } + } + self.expectThat(v1, Equals(expected_v1)) + + def test__multiple_ethernet_interfaces_with_dns(self): + node = factory.make_Node() + vlan = factory.make_VLAN() + subnet = factory.make_Subnet( + cidr='10.0.0.0/24', gateway_ip='10.0.0.1', + dns_servers=['10.0.0.2']) + subnet2 = factory.make_Subnet( + cidr='10.0.1.0/24', gateway_ip='10.0.1.1', + dns_servers=['10.0.1.2']) + eth0 = factory.make_Interface( + node=node, name='eth0', mac_address="00:01:02:03:04:05", vlan=vlan) + eth1 = factory.make_Interface( + node=node, name='eth1', mac_address="02:01:02:03:04:05") + node.boot_interface = eth0 + node.save() + factory.make_StaticIPAddress( + interface=eth0, subnet=subnet, ip='10.0.0.4', + alloc_type=IPADDRESS_TYPE.STICKY) + factory.make_StaticIPAddress( + interface=eth1, subnet=subnet2, ip='10.0.1.4', + alloc_type=IPADDRESS_TYPE.STICKY) + # Make sure we know when and where the default DNS server will be used. + get_default_dns_servers_mock = self.patch( + node, 'get_default_dns_servers') + get_default_dns_servers_mock.return_value = ['127.0.0.2'] + netplan = self._render_netplan_dict(node) + expected_netplan = { + 'network': { + 'version': 2, + 'ethernets': { + 'eth0': { + 'gateway': '10.0.0.1', + 'nameservers': { + 'addresses': ['10.0.0.2'] + }, + 'match': {'macaddress': '00:01:02:03:04:05'}, + 'mtu': 1500, + 'set-name': 'eth0', + 'addresses': ['10.0.0.4/24'], + }, + 'eth1': { + 'match': {'macaddress': '02:01:02:03:04:05'}, + 'nameservers': { + 'addresses': ['10.0.1.2'] + }, + 'mtu': 1500, + 'set-name': 'eth1', + 'addresses': ['10.0.1.4/24'], + }, + }, + }, + } + self.expectThat(netplan, Equals(expected_netplan)) + v1 = self._render_v1_dict(node) + expected_v1 = { + 'network': { + 'version': 1, + 'config': [ + { + 'id': 'eth0', + 'mac_address': '00:01:02:03:04:05', + 'mtu': 1500, + 'name': 'eth0', + 'subnets': [{ + 'address': '10.0.0.4/24', + 'dns_nameservers': ['10.0.0.2'], + 'gateway': '10.0.0.1', + 'type': 'static', + }], + 'type': 'physical' + }, + { + 'id': 'eth1', + 'mac_address': '02:01:02:03:04:05', + 'mtu': 1500, + 'name': 'eth1', + 'subnets': [{ + 'address': '10.0.1.4/24', + 'dns_nameservers': ['10.0.1.2'], + 'type': 'static', + }], + 'type': 'physical' + }, + { + 'address': ['127.0.0.2'], + 'search': ['maas'], + 'type': 'nameserver' + } + ], + } + } + self.expectThat(v1, Equals(expected_v1)) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_preseed.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_preseed.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_preseed.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_preseed.py 2018-08-17 02:41:34.000000000 +0000 @@ -1539,10 +1539,28 @@ 'get_boot_images_for').return_value = [] self.assertRaises(MissingBootImage, get_curtin_image, node) - def test_get_curtin_image_returns_xinstall_image(self): - node = factory.make_Node() + def test_get_curtin_image_returns_xinstall_image_for_subarch(self): + arch = factory.make_name('arch') + subarch = factory.make_name('subarch') + node = factory.make_Node(architecture=('%s/%s' % (arch, subarch))) other_images = [make_rpc_boot_image() for _ in range(3)] - xinstall_image = make_rpc_boot_image(purpose='xinstall') + xinstall_image = make_rpc_boot_image( + purpose='xinstall', architecture=arch, subarchitecture=subarch) + other_xinstall_image = make_rpc_boot_image( + purpose='xinstall', architecture=arch) + images = other_images + [xinstall_image, other_xinstall_image] + self.patch( + preseed_module, + 'get_boot_images_for').return_value = images + self.assertEqual(xinstall_image, get_curtin_image(node)) + + def test_get_curtin_image_returns_xinstall_image_for_newer(self): + arch = factory.make_name('arch') + subarch = factory.make_name('subarch') + node = factory.make_Node(architecture=('%s/%s' % (arch, subarch))) + other_images = [make_rpc_boot_image() for _ in range(3)] + xinstall_image = make_rpc_boot_image( + purpose='xinstall', architecture=arch) images = other_images + [xinstall_image] self.patch( preseed_module, @@ -1555,6 +1573,36 @@ architecture = make_usable_architecture(self) xinstall_path = factory.make_name('xi_path') xinstall_type = factory.make_name('xi_type') + cluster_ip = factory.make_ipv4_address() + node = factory.make_Node_with_Interface_on_Subnet( + primary_rack=self.rpc_rack_controller, osystem=osystem['name'], + architecture=architecture, distro_series=series, + boot_cluster_ip=cluster_ip) + arch, subarch = architecture.split('/') + boot_image = make_rpc_boot_image( + osystem=osystem['name'], release=series, + architecture=arch, subarchitecture=subarch, + purpose='xinstall', xinstall_path=xinstall_path, + xinstall_type=xinstall_type) + self.patch( + preseed_module, + 'get_boot_images_for').return_value = [boot_image] + + installer_url = get_curtin_installer_url(node) + self.assertEqual( + '%s:http://%s:5248/images/%s/%s/%s/%s/%s/%s' % ( + xinstall_type, cluster_ip, osystem['name'], arch, subarch, + series, boot_image['label'], xinstall_path), + installer_url) + + # XXX: roaksoax LP: #1739761 - Deploying precise is now done using + # the commissioning ephemeral environment. + def test_get_curtin_installer_url_returns_fsimage_precise_squashfs(self): + osystem = make_usable_osystem(self) + series = 'precise' + architecture = make_usable_architecture(self) + xinstall_path = factory.make_name('xi_path') + xinstall_type = factory.make_name('xi_type') cluster_ip = factory.make_ipv4_address() node = factory.make_Node_with_Interface_on_Subnet( primary_rack=self.rpc_rack_controller, osystem=osystem['name'], diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_preseed_storage.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_preseed_storage.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_preseed_storage.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_preseed_storage.py 2018-08-17 02:41:34.000000000 +0000 @@ -1506,3 +1506,369 @@ node._create_acquired_filesystems() config = compose_curtin_storage_config(node) self.assertStorageConfig(self.STORAGE_CONFIG, config) + + +class TestBootableRaidLayoutMBR(MAASServerTestCase, AssertStorageConfigMixin): + + STORAGE_CONFIG = dedent("""\ + config: + - id: sda + model: vendor + name: sda + ptable: msdos + serial: serial-a + type: disk + wipe: superblock + grub_device: true + - grub_device: true + id: sdb + model: vendor + name: sdb + ptable: gpt + serial: serial-b + type: disk + wipe: superblock + - device: sdb + flag: bios_grub + id: sdb-part1 + number: 1 + offset: 4194304B + size: 1048576B + type: partition + wipe: zero + - grub_device: true + id: sdc + model: vendor + name: sdc + ptable: gpt + serial: serial-c + type: disk + wipe: superblock + - device: sdc + flag: bios_grub + id: sdc-part1 + number: 1 + offset: 4194304B + size: 1048576B + type: partition + wipe: zero + - device: sda + flag: boot + id: sda-part1 + name: sda-part1 + number: 1 + size: 1099503239168B + type: partition + uuid: uuid-a + wipe: superblock + - device: sdb + flag: boot + id: sdb-part2 + name: sdb-part2 + number: 2 + size: 1099503239168B + type: partition + uuid: uuid-b + wipe: superblock + - device: sdc + flag: boot + id: sdc-part2 + name: sdc-part2 + number: 2 + size: 1099503239168B + type: partition + uuid: uuid-c + wipe: superblock + - devices: + - sda-part1 + - sdb-part2 + - sdc-part2 + id: md0 + name: md0 + raidlevel: 1 + spare_devices: [] + type: raid + - fstype: ext4 + id: md0_format + label: null + type: format + uuid: root-part + volume: md0 + - device: md0_format + id: md0_mount + path: / + type: mount + """) + + def test__renders_expected_output(self): + node = factory.make_Node( + status=NODE_STATUS.ALLOCATED, architecture='amd64/generic', + with_boot_disk=False) + terabyte = 1 * 1024 ** 4 + partitions = [] + for letter in 'abc': + disk = factory.make_PhysicalBlockDevice( + node=node, model='vendor', serial='serial-' + letter, + size=terabyte, name='sd' + letter) + table_type = ( + PARTITION_TABLE_TYPE.MBR if letter == 'a' + else PARTITION_TABLE_TYPE.GPT) + part_table = factory.make_PartitionTable( + table_type=table_type, block_device=disk) + partitions.append( + factory.make_Partition( + partition_table=part_table, + uuid='uuid-' + letter, + size=terabyte - PARTITION_TABLE_EXTRA_SPACE, + bootable=True)) + raid = RAID.objects.create_raid( + level=FILESYSTEM_GROUP_TYPE.RAID_1, + name="md0", uuid='uuid-raid', + partitions=partitions) + factory.make_Filesystem( + block_device=raid.virtual_device, fstype=FILESYSTEM_TYPE.EXT4, + uuid="root-part", mount_point="/", mount_options=None) + node._create_acquired_filesystems() + config = compose_curtin_storage_config(node) + self.assertStorageConfig(self.STORAGE_CONFIG, config) + + +class TestBootableRaidLayoutUEFI(MAASServerTestCase, AssertStorageConfigMixin): + + STORAGE_CONFIG = dedent("""\ + config: + - id: sda + model: vendor + name: sda + ptable: gpt + serial: serial-a + type: disk + wipe: superblock + grub_device: true + - grub_device: true + id: sdb + model: vendor + name: sdb + ptable: gpt + serial: serial-b + type: disk + wipe: superblock + - grub_device: true + id: sdc + model: vendor + name: sdc + ptable: gpt + serial: serial-c + type: disk + wipe: superblock + - device: sda + flag: boot + id: sda-part1 + name: sda-part1 + number: 1 + offset: 4194304B + size: 5497549750272B + type: partition + uuid: uuid-a + wipe: superblock + - device: sdb + flag: boot + id: sdb-part1 + name: sdb-part1 + number: 1 + offset: 4194304B + size: 5497549750272B + type: partition + uuid: uuid-b + wipe: superblock + - device: sdc + flag: boot + id: sdc-part1 + name: sdc-part1 + number: 1 + offset: 4194304B + size: 5497549750272B + type: partition + uuid: uuid-c + wipe: superblock + - devices: + - sda-part1 + - sdb-part1 + - sdc-part1 + id: md0 + name: md0 + raidlevel: 1 + spare_devices: [] + type: raid + - fstype: ext4 + id: md0_format + label: null + type: format + uuid: root-part + volume: md0 + - device: md0_format + id: md0_mount + path: / + type: mount + """) + + def test__renders_expected_output(self): + node = factory.make_Node( + status=NODE_STATUS.ALLOCATED, architecture="amd64/generic", + bios_boot_method='uefi', with_boot_disk=False) + size = 5 * 1024 ** 4 # 5Tb + partitions = [] + for letter in 'abc': + disk = factory.make_PhysicalBlockDevice( + node=node, model='vendor', serial='serial-' + letter, + size=size, name='sd' + letter) + table_type = PARTITION_TABLE_TYPE.GPT + part_table = factory.make_PartitionTable( + table_type=table_type, block_device=disk) + partitions.append( + factory.make_Partition( + partition_table=part_table, + uuid='uuid-' + letter, + size=size - PARTITION_TABLE_EXTRA_SPACE, + bootable=True)) + raid = RAID.objects.create_raid( + level=FILESYSTEM_GROUP_TYPE.RAID_1, + name="md0", uuid='uuid-raid', + partitions=partitions) + factory.make_Filesystem( + block_device=raid.virtual_device, fstype=FILESYSTEM_TYPE.EXT4, + uuid="root-part", mount_point="/", mount_options=None) + node._create_acquired_filesystems() + config = compose_curtin_storage_config(node) + self.assertStorageConfig(self.STORAGE_CONFIG, config) + + +class TestBootableRaidLayoutGPT(MAASServerTestCase, AssertStorageConfigMixin): + + STORAGE_CONFIG = dedent("""\ + config: + - id: sda + model: vendor + name: sda + ptable: gpt + serial: serial-a + type: disk + wipe: superblock + grub_device: true + - device: sda + flag: bios_grub + id: sda-part1 + number: 1 + offset: 4194304B + size: 1048576B + type: partition + wipe: zero + - grub_device: true + id: sdb + model: vendor + name: sdb + ptable: gpt + serial: serial-b + type: disk + wipe: superblock + - device: sdb + flag: bios_grub + id: sdb-part1 + number: 1 + offset: 4194304B + size: 1048576B + type: partition + wipe: zero + - grub_device: true + id: sdc + model: vendor + name: sdc + ptable: gpt + serial: serial-c + type: disk + wipe: superblock + - device: sdc + flag: bios_grub + id: sdc-part1 + number: 1 + offset: 4194304B + size: 1048576B + type: partition + wipe: zero + - device: sda + flag: boot + id: sda-part2 + name: sda-part2 + number: 2 + size: 5497549750272B + type: partition + uuid: uuid-a + wipe: superblock + - device: sdb + flag: boot + id: sdb-part2 + name: sdb-part2 + number: 2 + size: 5497549750272B + type: partition + uuid: uuid-b + wipe: superblock + - device: sdc + flag: boot + id: sdc-part2 + name: sdc-part2 + number: 2 + size: 5497549750272B + type: partition + uuid: uuid-c + wipe: superblock + - devices: + - sda-part2 + - sdb-part2 + - sdc-part2 + id: md0 + name: md0 + raidlevel: 1 + spare_devices: [] + type: raid + - fstype: ext4 + id: md0_format + label: null + type: format + uuid: root-part + volume: md0 + - device: md0_format + id: md0_mount + path: / + type: mount + """) + + def test__renders_expected_output(self): + node = factory.make_Node( + status=NODE_STATUS.ALLOCATED, architecture="amd64/generic", + with_boot_disk=False) + size = 5 * 1024 ** 4 # 5Tb + partitions = [] + for letter in 'abc': + disk = factory.make_PhysicalBlockDevice( + node=node, model='vendor', serial='serial-' + letter, + size=size, name='sd' + letter) + table_type = PARTITION_TABLE_TYPE.GPT + part_table = factory.make_PartitionTable( + table_type=table_type, block_device=disk) + partitions.append( + factory.make_Partition( + partition_table=part_table, + uuid='uuid-' + letter, + size=size - PARTITION_TABLE_EXTRA_SPACE, + bootable=True)) + raid = RAID.objects.create_raid( + level=FILESYSTEM_GROUP_TYPE.RAID_1, + name="md0", uuid='uuid-raid', + partitions=partitions) + factory.make_Filesystem( + block_device=raid.virtual_device, fstype=FILESYSTEM_TYPE.EXT4, + uuid="root-part", mount_point="/", mount_options=None) + node._create_acquired_filesystems() + config = compose_curtin_storage_config(node) + self.assertStorageConfig(self.STORAGE_CONFIG, config) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_proxyconfig.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_proxyconfig.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_proxyconfig.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_proxyconfig.py 2018-08-17 02:41:34.000000000 +0000 @@ -7,6 +7,7 @@ import os from pathlib import Path +import random from crochet import wait_for from django.conf import settings @@ -154,6 +155,19 @@ @wait_for_reactor @inlineCallbacks + def test__with_new_maas_proxy_port_changes_port(self): + self.patch(settings, "PROXY_CONNECT", True) + port = random.randint(1, 65535) + yield deferToDatabase( + transactional(Config.objects.set_config), + "maas_proxy_port", port) + yield proxyconfig.proxy_update_config(reload_proxy=False) + with self.proxy_path.open() as proxy_file: + lines = [line.strip() for line in proxy_file.readlines()] + self.assertIn('http_port %s' % port, lines) + + @wait_for_reactor + @inlineCallbacks def test__calls_reloadService(self): self.patch(settings, "PROXY_CONNECT", True) yield deferToDatabase(self.make_subnet) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_server_address.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_server_address.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_server_address.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_server_address.py 2018-08-17 02:41:34.000000000 +0000 @@ -214,12 +214,36 @@ region_ips, Equals([ IPAddress("192.168.0.254"), + IPAddress("192.168.0.1"), IPAddress("192.168.0.2"), IPAddress("192.168.0.4"), ]) ) - def test__alternates_include_one_ip_address_per_region(self): + def test__alternates_do_not_contain_duplicate_for_maas_url_ip(self): + # See bug #1753493. (This tests to ensure we don't provide the same + # IP address from maas_url twice.) Also ensures that the IP address + # from maas_url comes first. + factory.make_Subnet(cidr='192.168.0.0/24') + maas_url = 'http://192.168.0.2/MAAS' + rack = factory.make_RackController(url=maas_url) + r1 = factory.make_RegionController() + factory.make_Interface(node=r1, ip='192.168.0.1') + r2 = factory.make_RegionController() + factory.make_Interface(node=r2, ip='192.168.0.2') + # Make the "current" region controller r1. + self.patch(server_address, 'get_maas_id').return_value = r1.system_id + region_ips = get_maas_facing_server_addresses( + rack, include_alternates=True) + self.assertThat( + region_ips, + Equals([ + IPAddress("192.168.0.2"), + IPAddress("192.168.0.1"), + ]) + ) + + def test__alternates_include_one_ip_address_per_region_and_maas_url(self): factory.make_Subnet(cidr='192.168.0.0/24') maas_url = 'http://192.168.0.254/MAAS' rack = factory.make_RackController(url=maas_url) @@ -240,6 +264,7 @@ region_ips, Equals([ IPAddress("192.168.0.254"), + IPAddress("192.168.0.1"), IPAddress("192.168.0.2"), IPAddress("192.168.0.4"), ]) @@ -296,12 +321,12 @@ self.assertThat( region_ips, Equals([ - IPAddress("2001:db8::1"), IPAddress("192.168.0.1"), - IPAddress("2001:db8::2"), - IPAddress("2001:db8::4"), + IPAddress("2001:db8::1"), IPAddress("192.168.0.2"), IPAddress("192.168.0.4"), + IPAddress("2001:db8::2"), + IPAddress("2001:db8::4"), ]) ) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/tests/test_stats.py maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_stats.py --- maas-2.3.0-6434-gd354690/src/maasserver/tests/test_stats.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/tests/test_stats.py 2018-08-17 02:41:34.000000000 +0000 @@ -13,6 +13,7 @@ from maasserver.models import Config from maasserver.stats import ( get_maas_stats, + get_machine_stats, get_request_params, make_maas_user_agent_request, ) @@ -40,10 +41,18 @@ factory.make_RegionRackController() factory.make_RegionController() factory.make_RackController() - factory.make_Machine() + factory.make_Machine(cpu_count=2, memory=200) + factory.make_Machine(cpu_count=3, memory=100) factory.make_Device() stats = get_maas_stats() + machine_stats = get_machine_stats() + + # Due to floating point calculation subtleties, sometimes the value the + # database returns is off by one compared to the value Python + # calculates, so just get it directly from the database for the test. + total_storage = machine_stats['total_storage'] + compare = { "controllers": { "regionracks": 1, @@ -51,11 +60,16 @@ "racks": 1, }, "nodes": { - "machines": 1, + "machines": 2, "devices": 1, }, + "machine_stats": { + "total_cpu": 5, + "total_mem": 300, + "total_storage": total_storage, + }, } - self.assertEquals(stats, json.dumps(compare)) + self.assertEquals(json.loads(stats), compare) def test_get_request_params_returns_params(self): factory.make_RegionRackController() diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/triggers/system.py maas-2.3.5-6511-gf466fdb/src/maasserver/triggers/system.py --- maas-2.3.0-6434-gd354690/src/maasserver/triggers/system.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/triggers/system.py 2018-08-17 02:41:34.000000000 +0000 @@ -1590,6 +1590,7 @@ RETURNS trigger as $$ BEGIN IF (NEW.name = 'enable_proxy' OR + NEW.name = 'maas_proxy_port' OR NEW.name = 'use_peer_proxy' OR NEW.name = 'http_proxy') THEN PERFORM pg_notify('sys_proxy', ''); @@ -1606,6 +1607,7 @@ RETURNS trigger as $$ BEGIN IF (NEW.name = 'enable_proxy' OR + NEW.name = 'maas_proxy_port' OR NEW.name = 'use_peer_proxy' OR NEW.name = 'http_proxy') THEN PERFORM pg_notify('sys_proxy', ''); diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/triggers/tests/test_system_listener.py maas-2.3.5-6511-gf466fdb/src/maasserver/triggers/tests/test_system_listener.py --- maas-2.3.0-6434-gd354690/src/maasserver/triggers/tests/test_system_listener.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/triggers/tests/test_system_listener.py 2018-08-17 02:41:34.000000000 +0000 @@ -1436,7 +1436,7 @@ end_ip = str(IPAddress(network.first + 3)) yield deferToDatabase(self.create_iprange, { "subnet": subnet, - "type": IPRANGE_TYPE.DYNAMIC, + "alloc_type": IPRANGE_TYPE.DYNAMIC, "start_ip": start_ip, "end_ip": end_ip, }) @@ -1471,7 +1471,7 @@ end_ip = str(IPAddress(network.first + 3)) ip_range = yield deferToDatabase(self.create_iprange, { "subnet": subnet, - "type": IPRANGE_TYPE.DYNAMIC, + "alloc_type": IPRANGE_TYPE.DYNAMIC, "start_ip": start_ip, "end_ip": end_ip, }) @@ -1522,7 +1522,7 @@ end_ip = str(IPAddress(network.first + 3)) ip_range = yield deferToDatabase(self.create_iprange, { "subnet": subnet, - "type": IPRANGE_TYPE.DYNAMIC, + "alloc_type": IPRANGE_TYPE.DYNAMIC, "start_ip": start_ip, "end_ip": end_ip, }) @@ -1572,7 +1572,7 @@ end_ip = str(IPAddress(network.first + 3)) ip_range = yield deferToDatabase(self.create_iprange, { "subnet": subnet, - "type": IPRANGE_TYPE.RESERVED, + "alloc_type": IPRANGE_TYPE.RESERVED, "start_ip": start_ip, "end_ip": end_ip, }) @@ -1622,7 +1622,7 @@ end_ip = str(IPAddress(network.first + 3)) ip_range = yield deferToDatabase(self.create_iprange, { "subnet": subnet, - "type": IPRANGE_TYPE.DYNAMIC, + "alloc_type": IPRANGE_TYPE.DYNAMIC, "start_ip": start_ip, "end_ip": end_ip, }) @@ -4258,6 +4258,21 @@ @wait_for_reactor @inlineCallbacks + def test_sends_message_for_config_insert_maas_proxy_port(self): + yield deferToDatabase(register_system_triggers) + dv = DeferredValue() + listener = self.make_listener_without_delay() + listener.register( + "sys_proxy", lambda *args: dv.set(args)) + yield listener.startService() + try: + yield deferToDatabase(self.create_config, "maas_proxy_port", 9000) + yield dv.get(timeout=2) + finally: + yield listener.stopService() + + @wait_for_reactor + @inlineCallbacks def test_sends_message_for_config_insert_http_proxy(self): yield deferToDatabase(register_system_triggers) dv = DeferredValue() @@ -4303,6 +4318,22 @@ yield dv.get(timeout=2) finally: yield listener.stopService() + + @wait_for_reactor + @inlineCallbacks + def test_sends_message_for_config_update_maas_proxy_port(self): + yield deferToDatabase(register_system_triggers) + yield deferToDatabase(self.create_config, "maas_proxy_port", 8000) + dv = DeferredValue() + listener = self.make_listener_without_delay() + listener.register( + "sys_proxy", lambda *args: dv.set(args)) + yield listener.startService() + try: + yield deferToDatabase(self.set_config, "maas_proxy_port", 9000) + yield dv.get(timeout=2) + finally: + yield listener.stopService() @wait_for_reactor @inlineCallbacks diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/triggers/tests/test_websocket_listener.py maas-2.3.5-6511-gf466fdb/src/maasserver/triggers/tests/test_websocket_listener.py --- maas-2.3.0-6434-gd354690/src/maasserver/triggers/tests/test_websocket_listener.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/triggers/tests/test_websocket_listener.py 2018-08-17 02:41:34.000000000 +0000 @@ -31,6 +31,7 @@ from maasserver.utils.orm import transactional from maasserver.utils.threads import deferToDatabase from metadataserver.enum import SCRIPT_STATUS +from netaddr import IPAddress from provisioningserver.utils.twisted import ( asynchronous, DeferredValue, @@ -1695,10 +1696,17 @@ listener = self.make_listener_without_delay() dv = DeferredValue() listener.register("iprange", lambda *args: dv.set(args)) + network = factory.make_ipv4_network() + subnet = yield deferToDatabase( + self.create_subnet, {'cidr': str(network)}) + params = { + 'subnet': subnet, + 'start_ip': str(IPAddress(network.first + 2)), + 'end_ip': str(IPAddress(network.last - 1))} yield listener.startService() try: iprange = yield deferToDatabase( - self.create_iprange) + self.create_iprange, params) yield dv.get(timeout=2) self.assertEqual(('create', '%s' % iprange.id), dv.value) finally: @@ -3424,7 +3432,7 @@ try: iprange = yield deferToDatabase( self.create_iprange, { - "type": IPRANGE_TYPE.DYNAMIC, + "alloc_type": IPRANGE_TYPE.DYNAMIC, "subnet": subnet, "start_ip": '192.168.0.100', "end_ip": '192.168.0.110', @@ -3472,7 +3480,7 @@ }) iprange = yield deferToDatabase( self.create_iprange, { - "type": IPRANGE_TYPE.DYNAMIC, + "alloc_type": IPRANGE_TYPE.DYNAMIC, "subnet": old_subnet, "start_ip": '192.168.0.100', "end_ip": '192.168.0.110', diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/utils/__init__.py maas-2.3.5-6511-gf466fdb/src/maasserver/utils/__init__.py --- maas-2.3.0-6434-gd354690/src/maasserver/utils/__init__.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/utils/__init__.py 2018-08-17 02:41:34.000000000 +0000 @@ -19,6 +19,7 @@ urlencode, urljoin, urlparse, + urlsplit, ) from maasserver.config import RegionConfiguration @@ -27,6 +28,7 @@ ClusterConfiguration, UUID_NOT_SET, ) +from provisioningserver.utils.network import get_source_address from provisioningserver.utils.url import compose_URL from provisioningserver.utils.version import get_maas_version_user_agent @@ -141,6 +143,18 @@ return user_agent +def get_host_without_port(http_host): + return urlsplit('http://%s/' % http_host).hostname + + +def get_request_host(request): + """Returns the Host header from the specified HTTP request.""" + request_host = request.META.get('HTTP_HOST') + if request_host is not None: + request_host = get_host_without_port(request_host) + return request_host + + def get_remote_ip(request): """Returns the IP address of the host that initiated the request.""" return request.META.get('REMOTE_ADDR') @@ -178,3 +192,15 @@ return func(*args, **kwargs) return call_with_lock return synchronise + + +def get_default_region_ip(request): + """Returns the default reply address for the given HTTP request.""" + request_host = get_request_host(request) + if request_host is not None: + return request_host + remote_ip = get_remote_ip(request) + default_region_ip = None + if remote_ip is not None: + default_region_ip = get_source_address(remote_ip) + return default_region_ip diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/utils/tests/test_utils.py maas-2.3.5-6511-gf466fdb/src/maasserver/utils/tests/test_utils.py --- maas-2.3.0-6434-gd354690/src/maasserver/utils/tests/test_utils.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/utils/tests/test_utils.py 2018-08-17 02:41:34.000000000 +0000 @@ -27,6 +27,8 @@ absolute_url_reverse, build_absolute_uri, find_rack_controller, + get_default_region_ip, + get_host_without_port, get_local_cluster_UUID, get_maas_user_agent, strip_domain, @@ -40,6 +42,7 @@ from provisioningserver.utils.version import get_maas_version_user_agent from testtools.matchers import ( Contains, + Equals, Not, ) @@ -223,9 +226,13 @@ self.assertEqual(uuid, get_local_cluster_UUID()) -def make_request(origin_ip): +def make_request(origin_ip, http_host=None): """Return a fake HTTP request with the given remote address.""" - return RequestFactory().post('/', REMOTE_ADDR=str(origin_ip)) + if http_host is None: + return RequestFactory().post('/', REMOTE_ADDR=str(origin_ip)) + else: + return RequestFactory().post( + '/', REMOTE_ADDR=str(origin_ip), HTTP_HOST=str(http_host)) class TestFindRackController(MAASServerTestCase): @@ -283,3 +290,52 @@ composed_user_agent = "%s/%s" % ( get_maas_version_user_agent(), Config.objects.get_config('uuid')) self.assertEquals(user_agent, composed_user_agent) + + +class TestGetHostWithoutPort(MAASTestCase): + + scenarios = ( + ("ipv4", { + 'host': '127.0.0.1', + 'expected': '127.0.0.1' + }), + ("ipv4-with-port", { + 'host': '127.0.0.1:1234', + 'expected': '127.0.0.1' + }), + ("ipv6", { + 'host': '[2001:db8::1:2:3:4]', + 'expected': '2001:db8::1:2:3:4' + }), + ("ipv6-with-port", { + 'host': '[2001:db8::1]:4567', + 'expected': '2001:db8::1' + }), + ("dns", { + 'host': 'maas.example.com', + 'expected': 'maas.example.com' + }), + ) + + def test__returns_expected_results(self): + self.assertThat( + get_host_without_port(self.host), Equals(self.expected)) + + +class TestGetDefaultRegionIP(MAASServerTestCase): + + def test__returns_source_ip_based_on_remote_ip_if_no_Host_header(self): + # Note: the source IP should resolve to the loopback interface here. + self.assertThat( + get_default_region_ip(make_request("127.0.0.2")), + Equals("127.0.0.1")) + + def test__returns_Host_header_if_available(self): + self.assertThat( + get_default_region_ip(make_request("127.0.0.1", "localhost")), + Equals("localhost")) + + def test__returns_Host_header_if_available_and_strips_port(self): + self.assertThat( + get_default_region_ip(make_request("127.0.0.1", "localhost:5240")), + Equals("localhost")) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/bootresource.py maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/bootresource.py --- maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/bootresource.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/bootresource.py 2018-08-17 02:41:34.000000000 +0000 @@ -40,6 +40,7 @@ LargeFile, Node, ) +from maasserver.utils import get_maas_user_agent from maasserver.utils.converters import human_readable_bytes from maasserver.utils.orm import transactional from maasserver.utils.threads import deferToDatabase @@ -64,7 +65,6 @@ callOut, FOREVER, ) -from provisioningserver.utils.version import get_maas_version_user_agent from twisted.internet.defer import Deferred @@ -794,7 +794,7 @@ try: descriptions = download_all_image_descriptions( [source], - user_agent=get_maas_version_user_agent()) + user_agent=get_maas_user_agent()) except Exception as error: raise HandlerError(str(error)) items = list(descriptions.items()) diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/iprange.py maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/iprange.py --- maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/iprange.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/iprange.py 2018-08-17 02:41:34.000000000 +0000 @@ -37,12 +37,6 @@ def dehydrate(self, obj, data, for_list=False): """Add extra fields to `data`.""" - if obj.subnet is not None: - data['vlan'] = obj.subnet.vlan_id - else: - data['vlan'] = None - if obj.user is None: - data["user_username"] = "" - else: - data["user_username"] = obj.user.username + data['vlan'] = None if obj.subnet is None else obj.subnet.vlan_id + data['user'] = '' if obj.user is None else obj.user.username return data diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/tests/test_bootresource.py maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/tests/test_bootresource.py --- maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/tests/test_bootresource.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/tests/test_bootresource.py 2018-08-17 02:41:34.000000000 +0000 @@ -27,6 +27,7 @@ MAASServerTestCase, MAASTransactionServerTestCase, ) +from maasserver.utils import get_maas_user_agent from maasserver.utils.converters import human_readable_bytes from maasserver.utils.orm import ( get_one, @@ -53,7 +54,6 @@ make_image_spec, set_resource, ) -from provisioningserver.utils.version import get_maas_version_user_agent from testtools.matchers import ( ContainsAll, HasLength, @@ -1038,7 +1038,7 @@ mock_download, MockCalledOnceWith( [expected_source], - user_agent=get_maas_version_user_agent())) + user_agent=get_maas_user_agent())) def test_raises_error_on_downloading_resources(self): owner = factory.make_admin() diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/tests/test_iprange.py maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/tests/test_iprange.py --- maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/tests/test_iprange.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/tests/test_iprange.py 2018-08-17 02:41:34.000000000 +0000 @@ -25,8 +25,7 @@ "start_ip": iprange.start_ip, "end_ip": iprange.end_ip, "comment": iprange.comment, - "user": iprange.user.id if iprange.user else None, - "user_username": iprange.user.username if iprange.user else "", + "user": iprange.user.username if iprange.user else "", "type": iprange.type, "vlan": iprange.subnet.vlan_id if iprange.subnet else None } diff -Nru maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/vlan.py maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/vlan.py --- maas-2.3.0-6434-gd354690/src/maasserver/websockets/handlers/vlan.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/maasserver/websockets/handlers/vlan.py 2018-08-17 02:41:34.000000000 +0000 @@ -149,7 +149,7 @@ "end_ip": str(end_ipaddr), "type": IPRANGE_TYPE.DYNAMIC, "subnet": subnet.id, - "user": self.user.id, + "user": self.user.username, "comment": "Added via 'Provide DHCP...' in Web UI." }) iprange_form.save() diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/api.py maas-2.3.5-6511-gf466fdb/src/metadataserver/api.py --- maas-2.3.0-6434-gd354690/src/metadataserver/api.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/api.py 2018-08-17 02:41:34.000000000 +0000 @@ -72,7 +72,7 @@ ) from maasserver.utils import ( find_rack_controller, - get_remote_ip, + get_default_region_ip, ) from maasserver.utils.orm import ( get_one, @@ -102,7 +102,6 @@ EVENT_TYPES, ) from provisioningserver.logger import LegacyLogger -from provisioningserver.utils.network import get_source_address import yaml @@ -167,15 +166,6 @@ return get_node_for_mac(for_mac) -def get_default_region_ip(request): - """Returns the default reply address for the given HTTP request.""" - remote_ip = get_remote_ip(request) - default_region_ip = None - if remote_ip is not None: - default_region_ip = get_source_address(remote_ip) - return default_region_ip - - def make_text_response(contents): """Create a response containing `contents` as plain text.""" # XXX: Set a charset for text/plain. Django automatically encodes @@ -231,7 +221,9 @@ event_description="'%s' %s" % (origin, description), created=created) -def process_file(results, script_set, script_name, content, request): +def process_file( + results, script_set, script_name, content, request, + default_exit_status=None): """Process a file sent to MAAS over the metadata service.""" script_result_id = get_optional_param( @@ -287,7 +279,10 @@ if exit_status is not None: break if exit_status is None: - exit_status = 0 + if default_exit_status is None: + exit_status = 0 + else: + exit_status = default_exit_status results[script_result] = { 'exit_status': exit_status, diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/api_twisted.py maas-2.3.5-6511-gf466fdb/src/metadataserver/api_twisted.py --- maas-2.3.0-6434-gd354690/src/metadataserver/api_twisted.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/api_twisted.py 2018-08-17 02:41:34.000000000 +0000 @@ -304,6 +304,10 @@ activity_name = message['name'] description = message['description'] result = message.get('result', None) + # LP:1701352 - If no exit code is given by the client default to + # 0(pass) unless the signal is fail then set to 1(failure). This allows + # a Curtin failure to cause the ScriptResult to fail. + default_exit_status = 1 if result in ['FAIL', 'FAILURE'] else 0 # Add this event to the node event log. add_event_to_node_event_log( @@ -335,7 +339,9 @@ compression=sent_file.get('compression', None), encoding=sent_file['encoding'], content=sent_file['content']) - process_file(results, script_set, script_name, content, sent_file) + process_file( + results, script_set, script_name, content, sent_file, + default_exit_status) # Commit results to the database. for script_result, args in results.items(): diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/badblocks.py maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/badblocks.py --- maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/badblocks.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/badblocks.py 2018-08-17 02:41:34.000000000 +0000 @@ -31,6 +31,18 @@ # badblocks: # title: Bad blocks # description: The number of bad blocks found on the storage device. +# read_errors: +# title: Bad blocks read errors +# description: > +# The number of bad blocks read errors found on the storage device. +# write_errors: +# title: Bad blocks write errors +# description: > +# The number of bad blocks write errors found on the storage device. +# comparison_errors: +# title: Bad blocks comparison errors +# description: > +# The number of bad blocks comparison errors found on the storage device. # parameters: # storage: {type: storage} # destructive: {{if 'destructive' in name}}True{{else}}False{{endif}} @@ -113,37 +125,56 @@ if destructive: cmd = [ 'sudo', '-n', 'badblocks', '-b', str(blocksize), - '-c', str(parallel_blocks), '-v', '-f', '-w', storage + '-c', str(parallel_blocks), '-v', '-f', '-s', '-w', storage ] else: cmd = [ 'sudo', '-n', 'badblocks', '-b', str(blocksize), - '-c', str(parallel_blocks), '-v', '-f', '-n', storage + '-c', str(parallel_blocks), '-v', '-f', '-s', '-n', storage ] print('INFO: Running command: %s\n' % ' '.join(cmd)) proc = Popen(cmd, stdout=PIPE, stderr=STDOUT) stdout, _ = proc.communicate() + stdout = stdout.decode() # Print stdout to the console. - if stdout is not None: - print(stdout.decode()) + print(stdout) - result_path = os.environ.get("RESULT_PATH") + m = re.search( + '^Pass completed, (?P\d+) bad blocks found. ' + '\((?P\d+)\/(?P\d+)\/(?P\d+) errors\)$', + stdout, re.M) + badblocks = int(m.group('badblocks')) + read_errors = int(m.group('read')) + write_errors = int(m.group('write')) + comparison_errors = int(m.group('comparison')) + result_path = os.environ.get('RESULT_PATH') if result_path is not None: - # Parse the results for the desired information and - # then wrtie this to the results file. - match = re.search(b', ([0-9]+) bad blocks found', stdout) - if match is not None: - results = { - 'results': { - 'badblocks': int(match.group(1).decode()), - } + results = { + 'results': { + 'badblocks': badblocks, + 'read_errors': read_errors, + 'write_errors': write_errors, + 'comparison_errors': comparison_errors, } - with open(result_path, 'w') as results_file: - yaml.safe_dump(results, results_file) - - return proc.returncode + } + with open(result_path, 'w') as results_file: + yaml.safe_dump(results, results_file) + + # LP: #1733923 - Badblocks normally returns 0 no matter the result. If any + # errors are found fail the test. + if (proc.returncode + badblocks + read_errors + write_errors + + comparison_errors) != 0: + print('FAILURE: Test FAILED!') + print('INFO: %s badblocks found' % badblocks) + print('INFO: %s read errors found' % read_errors) + print('INFO: %s write errors found' % write_errors) + print('INFO: %s comparison errors found' % comparison_errors) + return 1 + else: + print('SUCCESS: Test PASSED!') + return 0 if __name__ == '__main__': diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/memtester.sh maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/memtester.sh --- maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/memtester.sh 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/memtester.sh 2018-08-17 02:41:34.000000000 +0000 @@ -1,10 +1,10 @@ -#!/bin/bash -e +#!/bin/bash -ex # # memtester - Run memtester against all available userspace memory. # # Author: Lee Trager # -# Copyright (C) 2017 Canonical +# Copyright (C) 2017-2018 Canonical # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -28,16 +28,15 @@ # packages: {apt: memtester} # --- End MAAS 1.0 script metadata --- -# Memtester can only test memory free to userspace. As a minimum, reserve -# the min_free_kbytes or the value of 0.77% of the memory on systems to -# ensure machine doesn't fail due to the OOM killer. Only run memtester -# against available RAM once. +# Memtester can only test memory free to userspace. At a minimum, reserve +# the min_free_kbytes + 10M or 0.77% of available memory, which ever is +# higher. This ensures the test doesn't fail due to the OOM killer. Only run +# memtester against available RAM once. min_free_kbytes=$(cat /proc/sys/vm/min_free_kbytes) -reserve=$(awk '/MemTotal/ { print (($2 * 0.077) / 10) }' /proc/meminfo) -reserve=${reserve%.*} +reserve=$(awk '/MemTotal/ { print int(($2 * 0.0077)) }' /proc/meminfo) if [ $reserve -le $min_free_kbytes ]; then reserve=$(($min_free_kbytes + 10240)) fi -sudo -n memtester \ - $(awk -v reserve=$reserve '/MemFree/ { print ($2 - reserve) "K"}' /proc/meminfo) 1 +testable_memory=$(awk -v reserve=$reserve '/MemFree/ { print int($2 - reserve) "K"}' /proc/meminfo) +sudo -n memtester $testable_memory 1 diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/ntp.sh maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/ntp.sh --- maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/ntp.sh 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/ntp.sh 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # # ntp - Run ntp clock set to verify NTP connectivity. # @@ -35,9 +35,13 @@ source /etc/os-release +function has_bin() { + which $1 >/dev/null + echo $? +} + if [ $VERSION_ID == "14.04" ]; then - which ntpq >/dev/null - if [ $? -ne 0 ]; then + if [ $(has_bin ntpd) -ne 0 ]; then echo -en 'Warning: NTP configuration is not supported in Trusty. ' 1>&2 echo -en 'Running with the default NTP configuration.\n\n' 1>&2 sudo -n apt-get install -q -y ntp @@ -45,13 +49,23 @@ ntpq -np sudo -n service ntp stop sudo -n timeout 10 ntpd -gq - ret=$? sudo -n service ntp start -else +elif [ $(has_bin ntpd) -eq 0 ]; then + echo -en 'INFO: ntpd detected.\n\n' 1>&2 ntpq -np sudo -n systemctl stop ntp.service sudo -n timeout 10 ntpd -gq ret=$? sudo -n systemctl start ntp.service +elif [ $(has_bin chronyc) -eq 0 ]; then + echo -en 'INFO: chrony detected.\n\n' 1>&2 + chronyc status + chronyc sources +elif [ $(has_bin timedatectl) -eq 0 ]; then + echo -en 'INFO: timesyncd detected.\n\n' 1>&2 + timedatectl status + sudo -n systemctl status systemd-timesyncd.service +else + echo -en 'ERROR: Unable to detect NTP service!\n\n' 1>&2 + exit 1 fi -exit $ret diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/tests/test_badblocks.py maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/tests/test_badblocks.py --- maas-2.3.0-6434-gd354690/src/metadataserver/builtin_scripts/tests/test_badblocks.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/builtin_scripts/tests/test_badblocks.py 2018-08-17 02:41:34.000000000 +0000 @@ -9,7 +9,6 @@ import io import os import random -import re from subprocess import ( PIPE, STDOUT, @@ -18,22 +17,22 @@ from unittest.mock import ANY from maastesting.factory import factory -from maastesting.matchers import ( - MockCalledOnce, - MockCalledOnceWith, -) +from maastesting.matchers import MockCalledOnceWith from maastesting.testcase import MAASTestCase from metadataserver.builtin_scripts import badblocks import yaml BADBLOCKS = random.randint(0, 1000) +READ_ERRORS = random.randint(0, 1000) +WRITE_ERRORS = random.randint(0, 1000) +COMPARISON_ERRORS = random.randint(0, 1000) BADBLOCKS_OUTPUT = dedent(""" Checking for bad blocks in non-destructive read-write mode From block 0 to 5242879 Testing with random pattern: - Pass completed, %s bad blocks found. (0/0/%s errors) - """ % (BADBLOCKS, BADBLOCKS)) + Pass completed, %s bad blocks found. (%s/%s/%s errors) + """ % (BADBLOCKS, READ_ERRORS, WRITE_ERRORS, COMPARISON_ERRORS)) class TestRunBadBlocks(MAASTestCase): @@ -75,7 +74,7 @@ }) cmd = [ 'sudo', '-n', 'badblocks', '-b', str(blocksize), - '-c', str(parallel_blocks), '-v', '-f', '-n', storage + '-c', str(parallel_blocks), '-v', '-f', '-s', '-n', storage ] mock_popen = self.patch(badblocks, "Popen") proc = mock_popen.return_value @@ -88,10 +87,13 @@ results = { 'results': { 'badblocks': BADBLOCKS, + 'read_errors': READ_ERRORS, + 'write_errors': WRITE_ERRORS, + 'comparison_errors': COMPARISON_ERRORS, } } - self.assertEquals(0, badblocks.run_badblocks(storage)) + self.assertEquals(1, badblocks.run_badblocks(storage)) self.assertThat(mock_popen, MockCalledOnceWith( cmd, stdout=PIPE, stderr=STDOUT)) self.assertThat(mock_open, MockCalledOnceWith(ANY, "w")) @@ -107,7 +109,7 @@ parallel_blocks) cmd = [ 'sudo', '-n', 'badblocks', '-b', str(blocksize), - '-c', str(parallel_blocks), '-v', '-f', '-w', storage, + '-c', str(parallel_blocks), '-v', '-f', '-s', '-w', storage, ] mock_popen = self.patch(badblocks, "Popen") proc = mock_popen.return_value @@ -116,27 +118,6 @@ proc.returncode = 0 self.assertEquals( - 0, badblocks.run_badblocks(storage, destructive=True)) + 1, badblocks.run_badblocks(storage, destructive=True)) self.assertThat(mock_popen, MockCalledOnceWith( cmd, stdout=PIPE, stderr=STDOUT)) - - def test_run_badblocks_exits_if_no_regex_match_found(self): - storage = factory.make_name('storage') - blocksize = random.randint(512, 4096) - self.patch(badblocks, 'get_block_size').return_value = blocksize - parallel_blocks = random.randint(1, 50000) - self.patch(badblocks, 'get_parallel_blocks').return_value = ( - parallel_blocks) - self.patch(os, "environ", { - "RESULT_PATH": factory.make_name() - }) - mock_popen = self.patch(badblocks, "Popen") - proc = mock_popen.return_value - proc.communicate.return_value = ( - BADBLOCKS_OUTPUT.encode('utf-8'), None) - proc.returncode = 0 - mock_re_search = self.patch(re, "search") - mock_re_search.return_value = None - - self.assertEquals(0, badblocks.run_badblocks(storage)) - self.assertThat(mock_re_search, MockCalledOnce()) diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/models/scriptset.py maas-2.3.5-6511-gf466fdb/src/metadataserver/models/scriptset.py --- maas-2.3.0-6434-gd354690/src/metadataserver/models/scriptset.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/models/scriptset.py 2018-08-17 02:41:34.000000000 +0000 @@ -253,6 +253,7 @@ # it is not associated with any disk. Check for this case and clean it # up when trying commissioning again. for script_result in ScriptResult.objects.filter( + script_set__result_type=new_script_set.result_type, script_set__node=node).exclude(parameters={}).exclude( script_set=new_script_set): for param in script_result.parameters.values(): diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/models/tests/test_scriptset.py maas-2.3.5-6511-gf466fdb/src/metadataserver/models/tests/test_scriptset.py --- maas-2.3.0-6434-gd354690/src/metadataserver/models/tests/test_scriptset.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/models/tests/test_scriptset.py 2018-08-17 02:41:34.000000000 +0000 @@ -16,6 +16,7 @@ Config, Event, EventType, + Node, ) from maasserver.preseed import CURTIN_INSTALL_LOG from maasserver.testing.factory import factory @@ -246,6 +247,28 @@ 1, ScriptResult.objects.filter(script_name=script_name).count()) + def test_create_commissioning_script_set_only_cleans_same_type(self): + # Regression test for LP:1751946 + node = Node.objects.create() + script_set = ScriptSet.objects.create_commissioning_script_set( + node=node) + script_set.scriptresult_set.update(status=SCRIPT_STATUS.ABORTED) + node.current_commissioning_script_set = script_set + script = factory.make_Script( + script_type=SCRIPT_TYPE.TESTING, + parameters={'storage': {'type': 'storage'}}) + testing_script_set = ScriptSet.objects.create_testing_script_set( + node=node, scripts=[script.id]) + testing_script_set.scriptresult_set.update( + status=SCRIPT_STATUS.ABORTED) + node.current_testing_script_set = testing_script_set + node.save() + new_script_set = ScriptSet.objects.create_commissioning_script_set( + node=node) + node.current_commissioning_script_set = new_script_set + node.save() + self.assertIsNotNone(ScriptSet.objects.get(id=testing_script_set.id)) + def test_create_commissioning_script_set_removes_previous_placeholder( self): # Regression test for LP:1731075 diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/tests/test_api.py maas-2.3.5-6511-gf466fdb/src/metadataserver/tests/test_api.py --- maas-2.3.0-6434-gd354690/src/metadataserver/tests/test_api.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/tests/test_api.py 2018-08-17 02:41:34.000000000 +0000 @@ -504,6 +504,28 @@ }, value) + def test_uses_default_exit_status_when_undef(self): + results = {} + script_result = factory.make_ScriptResult(status=SCRIPT_STATUS.RUNNING) + output = factory.make_string() + exit_status = random.randint(0, 255) + request = { + 'script_result_id': script_result.id, + 'script_version_id': script_result.script.script_id, + } + + process_file( + results, script_result.script_set, + factory.make_name('script_name'), output, request, exit_status) + + self.assertDictEqual( + { + 'exit_status': exit_status, + 'output': output, + 'script_version_id': script_result.script.script_id, + }, + results[script_result]) + def test_stores_script_version(self): results = {} script_name = factory.make_name('script_name') diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/tests/test_api_twisted.py maas-2.3.5-6511-gf466fdb/src/metadataserver/tests/test_api_twisted.py --- maas-2.3.0-6434-gd354690/src/metadataserver/tests/test_api_twisted.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/tests/test_api_twisted.py 2018-08-17 02:41:34.000000000 +0000 @@ -46,7 +46,10 @@ StatusHandlerResource, StatusWorkerService, ) -from metadataserver.enum import SCRIPT_STATUS +from metadataserver.enum import ( + RESULT_TYPE, + SCRIPT_STATUS, +) from metadataserver.models import NodeKey from testtools import ExpectedException from testtools.matchers import ( @@ -524,6 +527,37 @@ NODE_STATUS.FAILED_DEPLOYMENT, reload_object(node).status) self.assertIsNotNone(reload_object(node).owner) + def test_status_installation_failure_fails_script_result(self): + # Regression test for LP:1701352 + user = factory.make_User() + node = factory.make_Node( + interface=True, status=NODE_STATUS.DEPLOYING, owner=user) + node.current_installation_script_set = factory.make_ScriptSet( + node=node, result_type=RESULT_TYPE.INSTALLATION) + node.save() + script_result = factory.make_ScriptResult( + script_set=node.current_installation_script_set, + script_name=CURTIN_INSTALL_LOG, status=SCRIPT_STATUS.RUNNING) + content = factory.make_bytes() + payload = { + 'event_type': 'finish', + 'result': 'FAILURE', + 'origin': 'curtin', + 'name': 'cmd-install', + 'description': 'Command Install', + 'timestamp': datetime.utcnow(), + 'files': [ + { + "path": CURTIN_INSTALL_LOG, + "encoding": "base64", + "content": encode_as_base64(content), + } + ] + } + self.processMessage(node, payload) + self.assertEqual( + SCRIPT_STATUS.FAILED, reload_object(script_result).status) + def test_status_commissioning_failure_does_not_populate_tags(self): populate_tags_for_single_node = self.patch( api, "populate_tags_for_single_node") @@ -785,7 +819,7 @@ def test_status_with_results_no_exit_status_defaults_to_zero(self): """Adding a script result should succeed without a return code defaults - it to zero.""" + it to zero when passing.""" node = factory.make_Node( interface=True, status=NODE_STATUS.COMMISSIONING, with_empty_script_sets=True) @@ -797,7 +831,7 @@ encoded_content = encode_as_base64(bz2.compress(contents)) payload = { 'event_type': 'finish', - 'result': 'FAILURE', + 'result': 'OK', 'origin': 'curtin', 'name': 'commissioning', 'description': 'Commissioning', diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/commissioning.template maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/commissioning.template --- maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/commissioning.template 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/commissioning.template 2018-08-17 02:41:34.000000000 +0000 @@ -27,6 +27,10 @@ main() { prep_maas_api_helper + # LP:1730524 - Make sure we can signal the metadata service before updating + # the BMC username and password. + signal WORKING "Starting [maas-bmc-autodetect]" || exit 1 + # Install IPMI deps aptget install freeipmi-tools openipmi ipmitool sshpass @@ -61,7 +65,7 @@ if [ ! -z "$power_settings" ]; then signal \ "--power-type=${power_type}" "--power-parameters=${power_settings}" \ - WORKING "Finished [maas-ipmi-autodetect]" + WORKING "Finished [maas-bmc-autodetect]" fi maas-run-remote-scripts "--config=${CRED_CFG}" "${TEMP_D}" diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_api_helper.py maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_api_helper.py --- maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_api_helper.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_api_helper.py 2018-08-17 02:41:34.000000000 +0000 @@ -234,7 +234,11 @@ if None not in (power_type, power_params): params[b'power_type'] = power_type.encode('utf-8') - user, power_pass, power_address, driver = power_params.split(",") + if power_type == 'moonshot': + user, power_pass, power_address, driver = power_params.split(",") + else: + (user, power_pass, power_address, + driver, boot_type) = power_params.split(",") # OrderedDict is used to make testing easier. power_params = OrderedDict([ ('power_user', user), @@ -245,6 +249,7 @@ power_params['power_hwaddress'] = driver else: power_params['power_driver'] = driver + power_params['power_boot_type'] = boot_type params[b'power_parameters'] = json.dumps(power_params).encode() data, headers = encode_multipart_data( diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_enlist.sh maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_enlist.sh --- maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_enlist.sh 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_enlist.sh 2018-08-17 02:41:34.000000000 +0000 @@ -228,16 +228,14 @@ serverurl="maas.local" servername="$serverurl" fi -if echo "$serverurl" | egrep -q '(^[a-z]+://|^)[a-z0-9\.\-]+($|/$)'; then +if echo "$serverurl" | egrep -q '(^[a-z]+://|^)[a-zA-Z0-9\.\-]+($|/$)'; then api_url="MAAS/api/2.0/machines/" else api_url=`echo $serverurl | sed 's#^\(\|[a-z]\+://\)\([a-zA-Z0-9\.]\+\|\(\[[0-9a-fA-F:]\+\]\)\)\(\|\:[0-9]\+\)/##'` fi -#TODO: Auto-detect hostname? if [ -z "$hostname" ]; then - continue - #Error "No hostname has been provided" + echo "No hostname has been provided... MAAS will pick one automatically" fi if [ -z "$arch" ]; then diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py --- maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py 2018-08-17 02:41:34.000000000 +0000 @@ -263,16 +263,18 @@ run_command(('bmc-config', '--commit', '--filename', config)) -def get_maas_power_settings(user, password, ipaddress, version): - return "%s,%s,%s,%s" % (user, password, ipaddress, version) +def get_maas_power_settings(user, password, ipaddress, version, boot_type): + return "%s,%s,%s,%s,%s" % (user, password, ipaddress, version, boot_type) -def get_maas_power_settings_json(user, password, ipaddress, version): +def get_maas_power_settings_json( + user, password, ipaddress, version, boot_type): power_params = { "power_address": ipaddress, "power_pass": password, "power_user": user, "power_driver": version, + "power_boot_type": boot_type, } return json.dumps(power_params) @@ -301,8 +303,12 @@ # Create the extra characters to fullfill max_length letters += ''.join([ random.choice(string.ascii_letters) for _ in range(length - 7)]) - # Randomly mix the password - return ''.join(random.sample(letters, len(letters))) + # LP: #1758760 - Randomly mix the password until we ensure there's + # not consecutive occurrences of the same character. + password = ''.join(random.sample(letters, len(letters))) + while (bool(re.search(r'(.)\1', password))): + password = ''.join(random.sample(letters, len(letters))) + return password else: letters = string.ascii_letters + string.digits return ''.join([random.choice(letters) for _ in range(length)]) @@ -316,6 +322,13 @@ return False +def get_system_boot_type(): + """Detect if the system has boot EFI.""" + if os.path.isdir('/sys/firmware/efi'): + return 'efi' + return 'auto' + + def main(): import argparse @@ -374,12 +387,17 @@ IPMI_VERSION = "LAN_2_0" else: IPMI_VERSION = "LAN" + + IPMI_BOOT_TYPE = get_system_boot_type() + if args.commission_creds: print(get_maas_power_settings_json( - IPMI_MAAS_USER, IPMI_MAAS_PASSWORD, IPMI_IP_ADDRESS, IPMI_VERSION)) + IPMI_MAAS_USER, IPMI_MAAS_PASSWORD, IPMI_IP_ADDRESS, + IPMI_VERSION, IPMI_BOOT_TYPE)) else: print(get_maas_power_settings( - IPMI_MAAS_USER, IPMI_MAAS_PASSWORD, IPMI_IP_ADDRESS, IPMI_VERSION)) + IPMI_MAAS_USER, IPMI_MAAS_PASSWORD, IPMI_IP_ADDRESS, + IPMI_VERSION, IPMI_BOOT_TYPE)) if __name__ == '__main__': main() diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_run_remote_scripts.py maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_run_remote_scripts.py --- maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/maas_run_remote_scripts.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/maas_run_remote_scripts.py 2018-08-17 02:41:34.000000000 +0000 @@ -5,7 +5,7 @@ # # Author: Lee Trager # -# Copyright (C) 2017 Canonical +# Copyright (C) 2017-2018 Canonical # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -99,7 +99,9 @@ tar.extractall(scripts_dir) -def run_and_check(cmd, scripts, send_result=True): +def run_and_check(cmd, scripts, send_result=True, sudo=False): + if sudo: + cmd = ['sudo', '-En'] + cmd proc = Popen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE) capture_script_output( proc, scripts[0]['combined_path'], scripts[0]['stdout_path'], @@ -141,8 +143,8 @@ 'Installing apt packages for %s' % script['msg_name'], send_result, status='INSTALLING', **script['args']) if not run_and_check( - ['sudo', '-n', 'apt-get', '-qy', 'install'] + apt, - scripts, send_result): + ['apt-get', '-qy', 'install'] + apt, scripts, send_result, + True): return False if snap is not None: @@ -152,9 +154,9 @@ send_result, status='INSTALLING', **script['args']) for pkg in snap: if isinstance(pkg, str): - cmd = ['sudo', '-n', 'snap', 'install', pkg] + cmd = ['snap', 'install', pkg] elif isinstance(pkg, dict): - cmd = ['sudo', '-n', 'snap', 'install', pkg['name']] + cmd = ['snap', 'install', pkg['name']] if 'channel' in pkg: cmd.append('--%s' % pkg['channel']) if 'mode' in pkg: @@ -167,7 +169,7 @@ # string or dictionary. This should never happen but just # incase it does... continue - if not run_and_check(cmd, scripts, send_result): + if not run_and_check(cmd, scripts, send_result, True): return False if url is not None: @@ -207,16 +209,14 @@ elif filename.endswith('.deb'): # Allow dpkg to fail incase it just needs dependencies # installed. - run_and_check( - ['sudo', '-n', 'dpkg', '-i', filename], scripts, False) + run_and_check(['dpkg', '-i', filename], scripts, False, True) if not run_and_check( - ['sudo', '-n', 'apt-get', 'install', '-qyf'], scripts, - send_result): + ['apt-get', 'install', '-qyf'], scripts, send_result, + True): return False elif filename.endswith('.snap'): if not run_and_check( - ['sudo', '-n', 'snap', filename], scripts, - send_result): + ['snap', filename], scripts, send_result, True): return False # All went well, clean up the install logs so only script output is @@ -699,6 +699,11 @@ # running. os.nice(-20) + # Make sure installing packages is noninteractive for this process + # and all subprocesses. + if 'DEBIAN_FRONTEND' not in os.environ: + os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + heart_beat = HeartBeat(url, creds) heart_beat.start() diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/tests/test_maas_ipmi_autodetect.py maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/tests/test_maas_ipmi_autodetect.py --- maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/tests/test_maas_ipmi_autodetect.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/tests/test_maas_ipmi_autodetect.py 2018-08-17 02:41:34.000000000 +0000 @@ -6,6 +6,7 @@ __all__ = [] from collections import OrderedDict +import os.path import platform import re import subprocess @@ -31,6 +32,7 @@ format_user_key, generate_random_password, get_ipmi_ip_address, + get_system_boot_type, IPMIError, list_user_numbers, make_ipmi_user_settings, @@ -513,6 +515,10 @@ required_character_sets += 1 if required_character_sets < 2: return False + # Test password doesn't have two or more occurrences of the + # the same consecutive character. + if bool(re.search(r'(.)\1', password)): + return False return True max_attempts = 100 acceptable = 0 @@ -718,3 +724,24 @@ subprocess.CalledProcessError, bmc_set_mock) self.assertThat( bmc_set_mock.call_args_list, HasLength(1)) + + +class TestGetSystemBootType(MAASTestCase): + + def test_get_system_boot_type_efi(self): + """Test that returns .""" + boot_type = 'efi' + # path os.path.isdir to return True to simulate + # that /sys/firmware/efi exists. + self.patch(os.path, "isdir").return_value = True + actual_boot_type = get_system_boot_type() + self.assertEqual(boot_type, actual_boot_type) + + def test_get_system_boot_type_non_efi(self): + """Test """ + boot_type = 'auto' + # path os.path.isdir to return False to simulate + # that /sys/firmware/efi doesn't exist. + self.patch(os.path, "isdir").return_value = False + actual_boot_type = get_system_boot_type() + self.assertEqual(boot_type, actual_boot_type) diff -Nru maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/tests/test_maas_run_remote_scripts.py maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/tests/test_maas_run_remote_scripts.py --- maas-2.3.0-6434-gd354690/src/metadataserver/user_data/templates/snippets/tests/test_maas_run_remote_scripts.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/metadataserver/user_data/templates/snippets/tests/test_maas_run_remote_scripts.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2017 Canonical Ltd. This software is licensed under the +# Copyright 2017-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Tests for maas_run_remote_scripts.py.""" @@ -12,12 +12,17 @@ import os import random import stat -from subprocess import TimeoutExpired +from subprocess import ( + DEVNULL, + PIPE, + TimeoutExpired, +) import tarfile import time from unittest.mock import ( ANY, call, + MagicMock, ) from zipfile import ZipFile @@ -42,15 +47,16 @@ run_scripts_from_metadata, ) +# Unused ScriptResult id, used to make sure number is always unique. +SCRIPT_RESULT_ID = 0 + def make_script( scripts_dir=None, with_added_attribs=True, name=None, - script_result_id=None, script_version_id=None, timeout_seconds=None, - parallel=None, hardware_type=None, with_output=True): + script_version_id=None, timeout_seconds=None, parallel=None, + hardware_type=None, with_output=True): if name is None: name = factory.make_name('name') - if script_result_id is None: - script_result_id = random.randint(1, 1000) if script_version_id is None: script_version_id = random.randint(1, 1000) if timeout_seconds is None: @@ -59,6 +65,9 @@ parallel = random.randint(0, 2) if hardware_type is None: hardware_type = random.randint(0, 4) + global SCRIPT_RESULT_ID + script_result_id = SCRIPT_RESULT_ID + SCRIPT_RESULT_ID += 1 ret = { 'name': name, 'path': '%s/%s' % (random.choice(['commissioning', 'testing']), name), @@ -208,6 +217,16 @@ '%s\n%s\n' % (script['stdout'], script['stderr']), open(script['combined_path'], 'r').read()) + def test_sudo_run_and_check(self): + mock_popen = self.patch(maas_run_remote_scripts, 'Popen') + self.patch(maas_run_remote_scripts, 'capture_script_output') + cmd = factory.make_name('cmd') + + run_and_check([cmd], MagicMock(), False, True) + + self.assertThat(mock_popen, MockCalledOnceWith( + ['sudo', '-En', cmd], stdin=DEVNULL, stdout=PIPE, stderr=PIPE)) + def test_install_dependencies_does_nothing_when_empty(self): self.assertTrue(install_dependencies(make_scripts())) self.assertThat(self.mock_output_and_send, MockNotCalled()) @@ -227,8 +246,7 @@ 'Installing apt packages for %s' % script['msg_name'], True, status='INSTALLING')) self.assertThat(mock_run_and_check, MockCalledOnceWith( - ['sudo', '-n', 'apt-get', '-qy', 'install'] + packages, - scripts, True)) + ['apt-get', '-qy', 'install'] + packages, scripts, True, True)) # Verify cleanup self.assertFalse(os.path.exists(script['combined_path'])) self.assertFalse(os.path.exists(script['stdout_path'])) @@ -250,8 +268,7 @@ 'Installing apt packages for %s' % script['msg_name'], True, status='INSTALLING')) self.assertThat(mock_run_and_check, MockCalledOnceWith( - ['sudo', '-n', 'apt-get', '-qy', 'install'] + packages, - scripts, True)) + ['apt-get', '-qy', 'install'] + packages, scripts, True, True)) def test_install_dependencies_snap_str_list(self): mock_run_and_check = self.patch( @@ -274,8 +291,7 @@ for package in packages: self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'snap', 'install', package], - scripts, True)) + ['snap', 'install', package], scripts, True, True)) def test_install_dependencies_snap_str_dict(self): mock_run_and_check = self.patch( @@ -315,28 +331,27 @@ self.assertFalse(os.path.exists(script['stdout_path'])) self.assertFalse(os.path.exists(script['stderr_path'])) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'snap', 'install', packages[0]['name']], - scripts, True)) + ['snap', 'install', packages[0]['name']], scripts, True, True)) self.assertThat(mock_run_and_check, MockAnyCall( [ - 'sudo', '-n', 'snap', 'install', packages[1]['name'], + 'snap', 'install', packages[1]['name'], '--%s' % packages[1]['channel'] ], - scripts, True)) + scripts, True, True)) self.assertThat(mock_run_and_check, MockAnyCall( [ - 'sudo', '-n', 'snap', 'install', packages[2]['name'], + 'snap', 'install', packages[2]['name'], '--%s' % packages[2]['channel'], '--%smode' % packages[2]['mode'], ], - scripts, True)) + scripts, True, True)) self.assertThat(mock_run_and_check, MockAnyCall( [ - 'sudo', '-n', 'snap', 'install', packages[3]['name'], + 'snap', 'install', packages[3]['name'], '--%s' % packages[3]['channel'], '--%smode' % packages[3]['mode'], ], - scripts, True)) + scripts, True, True)) def test_install_dependencies_snap_errors(self): mock_run_and_check = self.patch( @@ -355,8 +370,7 @@ True, status='INSTALLING')) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'snap', 'install', packages[0]], - scripts, True)) + ['snap', 'install', packages[0]], scripts, True, True)) def test_install_dependencies_url(self): mock_run_and_check = self.patch( @@ -452,9 +466,9 @@ self.assertTrue(install_dependencies(scripts)) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'dpkg', '-i', deb_file], scripts, False)) + ['dpkg', '-i', deb_file], scripts, False, True)) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'apt-get', 'install', '-qyf'], scripts, True)) + ['apt-get', 'install', '-qyf'], scripts, True, True)) def test_install_dependencies_url_deb_errors(self): mock_run_and_check = self.patch( @@ -471,9 +485,9 @@ self.assertFalse(install_dependencies(scripts)) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'dpkg', '-i', deb_file], scripts, False)) + ['dpkg', '-i', deb_file], scripts, False, True)) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'apt-get', 'install', '-qyf'], scripts, True)) + ['apt-get', 'install', '-qyf'], scripts, True, True)) def test_install_dependencies_url_snap(self): mock_run_and_check = self.patch( @@ -489,7 +503,7 @@ self.assertTrue(install_dependencies(scripts)) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'snap', snap_file], scripts, True)) + ['snap', snap_file], scripts, True, True)) def test_install_dependencies_url_snap_errors(self): mock_run_and_check = self.patch( @@ -506,7 +520,7 @@ self.assertFalse(install_dependencies(scripts)) self.assertThat(mock_run_and_check, MockAnyCall( - ['sudo', '-n', 'snap', snap_file], scripts, True)) + ['snap', snap_file], scripts, True, True)) class TestParseParameters(MAASTestCase): @@ -867,7 +881,8 @@ single_thread = make_scripts(instance=False, parallel=0) instance_thread = [ make_scripts(parallel=1) - for _ in range(3)] + for _ in range(3) + ] any_thread = make_scripts(instance=False, parallel=2) scripts = copy.deepcopy(single_thread) for instance_thread_group in instance_thread: diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/boot/__init__.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/boot/__init__.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/boot/__init__.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/boot/__init__.py 2018-08-17 02:41:34.000000000 +0000 @@ -13,6 +13,7 @@ abstractproperty, ) from errno import ENOENT +from functools import lru_cache from io import BytesIO import os from typing import Dict @@ -283,10 +284,12 @@ isinstance(element, str) for element in self.bootloader_files) assert isinstance(self.arch_octet, str) or self.arch_octet is None + @lru_cache(1) def get_template_dir(self): """Gets the template directory for the boot method.""" return locate_template("%s" % self.template_subdir) + @lru_cache(512) def get_template(self, purpose, arch, subarch): """Gets the best avaliable template for the boot method. diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/boot/tests/test_uefi_amd64.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/boot/tests/test_uefi_amd64.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/boot/tests/test_uefi_amd64.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/boot/tests/test_uefi_amd64.py 2018-08-17 02:41:34.000000000 +0000 @@ -132,7 +132,7 @@ purpose="local", arch="amd64"), } output = method.get_reader(**options).read(10000).decode("utf-8") - self.assertIn("chainloader /efi/ubuntu/shimx64.efi", output) + self.assertIn("chainloader /efi/ubuntu/grubx64.efi", output) def test_get_reader_with_enlist_purpose(self): # If purpose is "enlist", the config.enlist.template should be diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/dhcp/config.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/dhcp/config.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/dhcp/config.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/dhcp/config.py 2018-08-17 02:41:34.000000000 +0000 @@ -28,7 +28,7 @@ get_path, ) from provisioningserver.utils import ( - locate_template, + load_template, snappy, typed, ) @@ -46,7 +46,7 @@ # Used to generate the conditional bootloader behaviour -CONDITIONAL_BOOTLOADER = (""" +CONDITIONAL_BOOTLOADER = tempita.Template(""" {{if ipv6}} {{behaviour}} exists dhcp6.client-arch-type and option dhcp6.client-arch-type = {{arch_octet}} { @@ -64,7 +64,7 @@ """) # Used to generate the PXEBootLoader special case -DEFAULT_BOOTLOADER = (""" +DEFAULT_BOOTLOADER = tempita.Template(""" {{if ipv6}} else { option dhcp6.bootfile-url \"{{url}}\"; @@ -94,8 +94,7 @@ if method.path_prefix: url += method.path_prefix url += '/%s' % method.bootloader_path - output += tempita.sub( - CONDITIONAL_BOOTLOADER, + output += CONDITIONAL_BOOTLOADER.substitute( ipv6=ipv6, rack_ip=rack_ip, url=url, behaviour=next(behaviour), arch_octet=method.arch_octet, @@ -114,8 +113,7 @@ if method.path_prefix: url += method.path_prefix url += '/%s' % method.bootloader_path - output += tempita.sub( - DEFAULT_BOOTLOADER, + output += DEFAULT_BOOTLOADER.substitute( ipv6=ipv6, rack_ip=rack_ip, url=url, bootloader=method.bootloader_path, path_prefix=method.path_prefix, @@ -262,8 +260,7 @@ """ bootloader = compose_conditional_bootloader(False) platform_codename = linux_distribution()[2] - template_file = locate_template('dhcp', template_name) - template = tempita.Template.from_filename(template_file, encoding="UTF-8") + template = load_template('dhcp', template_name) dhcp_socket = get_data_path('/var/lib/maas/dhcpd.sock') # Helper functions to stuff into the template namespace. @@ -318,8 +315,7 @@ :return: A full configuration, as a string. """ platform_codename = linux_distribution()[2] - template_file = locate_template('dhcp', template_name) - template = tempita.Template.from_filename(template_file, encoding="UTF-8") + template = load_template('dhcp', template_name) # Helper functions to stuff into the template namespace. helpers = { "oneline": normalise_whitespace, diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/dns/config.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/dns/config.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/dns/config.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/dns/config.py 2018-08-17 02:41:34.000000000 +0000 @@ -22,13 +22,12 @@ from provisioningserver.logger import get_maas_logger from provisioningserver.utils import ( + load_template, locate_config, - locate_template, ) from provisioningserver.utils.fs import atomic_write from provisioningserver.utils.isc import read_isc_file from provisioningserver.utils.shell import call_and_check -import tempita maaslog = get_maas_logger("dns") @@ -181,12 +180,6 @@ call_and_check(rndc_cmd) -def get_dns_template_path(template_name): - """Return the path to a dns template file.""" - # This function exists solely to make unit testing easier. - return locate_template('dns', template_name) - - def set_up_options_conf(overwrite=True, **kwargs): """Write out the named.conf.options.inside.maas file. @@ -195,10 +188,7 @@ so relies on either the DNSFixture in the test suite, or the packaging. Both should set that file up appropriately to include our file. """ - template_path = get_dns_template_path( - "named.conf.options.inside.maas.template") - template = tempita.Template.from_filename( - template_path, encoding="UTF-8") + template = load_template('dns', 'named.conf.options.inside.maas.template') # Make sure "upstream_dns" is set at least to None. It's a special piece # of config and we don't want to require that every call site has to @@ -257,8 +247,7 @@ :param parameters: One or more dicts of paramaters to be passed to the template. Each adds to (and may overwrite) the previous ones. """ - template_path = locate_template('dns', template_name) - template = tempita.Template.from_filename(template_path, encoding="UTF-8") + template = load_template('dns', template_name) combined_params = {} for params_dict in parameters: combined_params.update(params_dict) diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/dns/tests/test_config.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/dns/tests/test_config.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/dns/tests/test_config.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/dns/tests/test_config.py 2018-08-17 02:41:34.000000000 +0000 @@ -259,22 +259,6 @@ dns_conf_dir, MAAS_NAMED_CONF_OPTIONS_INSIDE_NAME) self.assertThat(target_file, FileExists()) - def test_template_path_is_correct(self): - template_path = config.get_dns_template_path("file") - expected = os.path.abspath( - os.path.join( - os.path.dirname(__file__), "../../templates/dns/file")) - self.assertEqual(expected, template_path) - - def test_set_up_options_conf_raises_on_bad_template(self): - template = self.make_file( - name="named.conf.options.inside.maas.template", - contents=b"{{nonexistent}}") - self.patch( - config, "get_dns_template_path").return_value = template - exception = self.assertRaises(DNSConfigFail, set_up_options_conf) - self.assertIn("name 'nonexistent' is not defined", repr(exception)) - def test_rndc_config_includes_default_controls(self): dns_conf_dir = patch_dns_config_path(self) patch_dns_default_controls(self, enable=True) diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/osystem/tests/test_ubuntu.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/osystem/tests/test_ubuntu.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/osystem/tests/test_ubuntu.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/osystem/tests/test_ubuntu.py 2018-08-17 02:41:34.000000000 +0000 @@ -19,7 +19,11 @@ class TestUbuntuOS(MAASTestCase): def get_lts_release(self): - return UbuntuDistroInfo().lts() + # XXX roaksoax 2017-04-26 LP: #1767137 + # 2.3 uses Xenial as the default release. Hardcode this to + # xenial to ensure it doesn't break CI environments or new + # installs that expect xenial. + return "xenial" def get_release_title(self, release): info = UbuntuDistroInfo() diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/osystem/ubuntu.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/osystem/ubuntu.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/osystem/ubuntu.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/osystem/ubuntu.py 2018-08-17 02:41:34.000000000 +0000 @@ -41,8 +41,13 @@ return row is not None def get_lts_release(self): - """Return the latest Ubuntu LTS release.""" - return self.ubuntu_distro_info.lts() + """Return the default Ubuntu LTS release for this MAAS release.""" + # XXX roaksoax 2017-04-26 LP: #1767137 - This function used to + # rely on distro info to get the default LTS. However, since it now + # returns a release that's not 'xenial', this breaks new installs + # or CI environments. As such, hard code the LTS release to use as + # a default here. + return "xenial" def get_default_release(self): """Gets the default release to use when a release is not diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/pod/rsd.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/pod/rsd.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/pod/rsd.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/pod/rsd.py 2018-08-17 02:41:34.000000000 +0000 @@ -428,7 +428,7 @@ discovered_pod.cores += sum(cores) discovered_pod.cpu_speeds.extend(cpu_speeds) # GiB to Bytes. - discovered_pod.local_storage += sum(storages) * 1073741824 + discovered_pod.local_storage += sum(storages) * (1024 ** 3) discovered_pod.local_disks += len(storages) # Set cpu_speed to max of all found cpu_speeds. diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/pod/tests/test_virsh.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/pod/tests/test_virsh.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/pod/tests/test_virsh.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/pod/tests/test_virsh.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2017 Canonical Ltd. This software is licensed under the +# Copyright 2017-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Tests for `provisioningserver.drivers.pod.virsh`.""" @@ -1006,6 +1006,8 @@ mock_run = self.patch(virsh.VirshSSH, "run") mock_attach_disk = self.patch(virsh.VirshSSH, "attach_local_volume") mock_attach_nic = self.patch(virsh.VirshSSH, "attach_interface") + mock_set_machine_autostart = self.patch( + virsh.VirshSSH, "set_machine_autostart") mock_configure_pxe = self.patch(virsh.VirshSSH, "configure_pxe_boot") mock_discovered = self.patch(virsh.VirshSSH, "get_discovered_machine") mock_discovered.return_value = sentinel.discovered @@ -1017,7 +1019,9 @@ self.assertThat( mock_attach_nic, MockCalledOnceWith(ANY, 'maas')) self.assertThat( - mock_configure_pxe, MockCalledOnceWith(ANY)) + mock_set_machine_autostart, MockCalledOnceWith(request.hostname)) + self.assertThat( + mock_configure_pxe, MockCalledOnceWith(request.hostname)) self.assertThat( mock_discovered, MockCalledOnceWith(ANY, request=request)) self.assertEquals(sentinel.discovered, observed) @@ -1032,7 +1036,6 @@ call([ 'undefine', domain, '--remove-all-storage', - '--delete-snapshots', '--managed-save']))) diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/pod/virsh.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/pod/virsh.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/pod/virsh.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/pod/virsh.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2017 Canonical Ltd. This software is licensed under the +# Copyright 2017-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Virsh pod driver.""" @@ -381,7 +381,7 @@ # Skip if cannot get more information. continue pools[pool] = convert_size_to_bytes( - self.get_key_value(output, "Capacity")) + self.get_key_value(output, key)) return pools def get_pod_local_storage(self): @@ -533,6 +533,14 @@ discovered_machine.interfaces = interfaces return discovered_machine + def set_machine_autostart(self, machine): + """Set machine to autostart.""" + output = self.run(['autostart', machine]).strip() + if output.startswith("error:"): + maaslog.error("%s: Failed to set autostart", machine) + return False + return True + def configure_pxe_boot(self, machine): """Given the specified machine, reads the XML dump and determines if the boot order needs to be changed. The boot order needs to be @@ -760,6 +768,9 @@ for _ in request.interfaces: self.attach_interface(request.hostname, best_network) + # Set machine to autostart. + self.set_machine_autostart(request.hostname) + # Setup the domain to PXE boot. self.configure_pxe_boot(request.hostname) @@ -771,9 +782,11 @@ # Ensure that its destroyed first. self.run(['destroy', domain]) # Undefine the domains and remove all storage and snapshots. + # XXX newell 2018-02-25 bug=1741165 + # Removed the --delete-snapshots flag to workaround the volumes not + # being deleted. See the bug for more details. self.run([ - 'undefine', domain, - '--remove-all-storage', '--delete-snapshots', '--managed-save']) + 'undefine', domain, '--remove-all-storage', '--managed-save']) class VirshPodDriver(PodDriver): diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/__init__.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/__init__.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/__init__.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/__init__.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2014-2016 Canonical Ltd. This software is licensed under the +# Copyright 2014-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Base power driver.""" @@ -384,6 +384,10 @@ # Wait before retrying. yield pause(waiting_time, self.clock) else: + # LP:1768659 - If the power driver isn't queryable(manual) + # checking the power state will always fail. + if not self.queryable: + return # Wait before checking state. yield pause(waiting_time, self.clock) # Try to get power state. diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/ipmi.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/ipmi.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/ipmi.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/ipmi.py 2018-08-17 02:41:34.000000000 +0000 @@ -39,6 +39,14 @@ """ +IPMI_CONFIG_WITH_BOOT_TYPE = """\ +Section Chassis_Boot_Flags + Boot_Flags_Persistent No + BIOS_Boot_Type %s + Boot_Device PXE +EndSection +""" + IPMI_ERRORS = { 'username invalid': { 'message': ( @@ -159,6 +167,26 @@ ] +class IPMI_BOOT_TYPE: + # DEFAULT used to provide backwards compatibility + DEFAULT = 'auto' + LEGACY = 'legacy' + EFI = 'efi' + + +IPMI_BOOT_TYPE_CHOICES = [ + [IPMI_BOOT_TYPE.DEFAULT, "Automatic"], + [IPMI_BOOT_TYPE.LEGACY, "Legacy boot"], + [IPMI_BOOT_TYPE.EFI, "EFI boot"] + ] + + +IPMI_BOOT_TYPE_MAPPING = { + IPMI_BOOT_TYPE.EFI: 'EFI', + IPMI_BOOT_TYPE.LEGACY: 'PC-COMPATIBLE', + } + + class IPMIPowerDriver(PowerDriver): name = 'ipmi' @@ -168,6 +196,11 @@ 'power_driver', "Power driver", field_type='choice', choices=IPMI_DRIVER_CHOICES, default=IPMI_DRIVER.LAN_2_0, required=True), + make_setting_field( + 'power_boot_type', "Power boot type", field_type='choice', + choices=IPMI_BOOT_TYPE_CHOICES, default=IPMI_BOOT_TYPE.DEFAULT, + required=False + ), make_setting_field('power_address', "IP address", required=True), make_setting_field('power_user', "Power user"), make_setting_field( @@ -185,11 +218,17 @@ @staticmethod def _issue_ipmi_chassis_config_command( - command, power_change, power_address): + command, power_change, power_address, power_boot_type=None): env = shell.select_c_utf8_locale() with NamedTemporaryFile("w+", encoding="utf-8") as tmp_config: # Write out the chassis configuration. - tmp_config.write(IPMI_CONFIG) + if (power_boot_type is None or + power_boot_type == IPMI_BOOT_TYPE.DEFAULT): + tmp_config.write(IPMI_CONFIG) + else: + tmp_config.write( + IPMI_CONFIG_WITH_BOOT_TYPE % + IPMI_BOOT_TYPE_MAPPING[power_boot_type]) tmp_config.flush() # Use it when running the chassis config command. # XXX: Not using call_and_check here because we @@ -234,7 +273,7 @@ def _issue_ipmi_command( self, power_change, power_address=None, power_user=None, power_pass=None, power_driver=None, power_off_mode=None, - mac_address=None, **extra): + mac_address=None, power_boot_type=None, **extra): """Issue command to ipmipower, for the given system.""" # This script deliberately does not check the current power state # before issuing the requested power command. See bug 1171418 for an @@ -263,18 +302,19 @@ common_args.extend(("-u", power_user)) common_args.extend(('-p', power_pass)) - # Update the chassis config and power commands. - ipmi_chassis_config_command.extend(common_args) - ipmi_chassis_config_command.append('--commit') + # Update the power commands with common args. ipmipower_command.extend(common_args) - # Before changing state run the chassis config command. - if power_change in ("on", "off"): - self._issue_ipmi_chassis_config_command( - ipmi_chassis_config_command, power_change, power_address) - # Additional arguments for the power command. if power_change == 'on': + # Update the chassis config commands and call it just when + # powering on the machine. + ipmi_chassis_config_command.extend(common_args) + ipmi_chassis_config_command.append('--commit') + self._issue_ipmi_chassis_config_command( + ipmi_chassis_config_command, power_change, power_address, + power_boot_type) + ipmipower_command.append('--cycle') ipmipower_command.append('--on-if-off') elif power_change == 'off': diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/tests/test_base.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/tests/test_base.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/tests/test_base.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/tests/test_base.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2014-2016 Canonical Ltd. This software is licensed under the +# Copyright 2014-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Tests for `provisioningserver.drivers.power`.""" @@ -403,6 +403,18 @@ with ExpectedException(PowerError): yield method(system_id, context) + @inlineCallbacks + def test_doesnt_power_query_if_unqueryable(self): + system_id = factory.make_name('system_id') + context = {'context': factory.make_name('context')} + driver = make_power_driver(wait_time=[0]) + driver.queryable = False + self.patch(driver, self.action_func) + mock_query = self.patch(driver, 'power_query') + method = getattr(driver, self.action) + yield method(system_id, context) + self.assertThat(mock_query, MockNotCalled()) + class TestPowerDriverCycle(MAASTestCase): diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/tests/test_ipmi.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/tests/test_ipmi.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/drivers/power/tests/test_ipmi.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/drivers/power/tests/test_ipmi.py 2018-08-17 02:41:34.000000000 +0000 @@ -25,7 +25,10 @@ PowerError, ) from provisioningserver.drivers.power.ipmi import ( + IPMI_BOOT_TYPE, + IPMI_BOOT_TYPE_MAPPING, IPMI_CONFIG, + IPMI_CONFIG_WITH_BOOT_TYPE, IPMI_ERRORS, IPMIPowerDriver, ) @@ -100,19 +103,9 @@ context['mac_address'] = factory.make_mac_address() context['power_address'] = random.choice((None, "", " ")) - self.patch_autospec(driver, "_issue_ipmi_chassis_config_command") self.patch_autospec(driver, "_issue_ipmipower_command") driver._issue_ipmi_command(power_change, **context) - # The IP address is passed to _issue_ipmi_chassis_config_command. - self.assertThat( - driver._issue_ipmi_chassis_config_command, - MockCalledOnceWith(ANY, power_change, ip_address)) - # The IP address is also within the command passed to - # _issue_ipmi_chassis_config_command. - self.assertThat( - driver._issue_ipmi_chassis_config_command.call_args[0], - Contains(ip_address)) # The IP address is passed to _issue_ipmipower_command. self.assertThat( driver._issue_ipmipower_command, @@ -228,23 +221,19 @@ def test__issue_ipmi_command_issues_power_off(self): context = make_context() - ipmi_chassis_config_command = make_ipmi_chassis_config_command( - **context, tmp_config_name=ANY) ipmipower_command = make_ipmipower_command(**context) ipmipower_command += ('--off', ) ipmi_power_driver = IPMIPowerDriver() env = select_c_utf8_locale() popen_mock = self.patch(ipmi_module, 'Popen') process = popen_mock.return_value - process.communicate.side_effect = [(b'', b''), (b'off', b'')] + process.communicate.side_effect = [(b'off', b'')] process.returncode = 0 result = ipmi_power_driver._issue_ipmi_command('off', **context) self.expectThat( popen_mock, MockCallsMatch( - call(ipmi_chassis_config_command, stdout=PIPE, - stderr=PIPE, env=env), call(ipmipower_command, stdout=PIPE, stderr=PIPE, env=env))) self.expectThat(result, Equals('off')) @@ -252,23 +241,19 @@ def test__issue_ipmi_command_issues_power_off_soft_mode(self): context = make_context() context['power_off_mode'] = 'soft' - ipmi_chassis_config_command = make_ipmi_chassis_config_command( - **context, tmp_config_name=ANY) ipmipower_command = make_ipmipower_command(**context) ipmipower_command += ('--soft', ) ipmi_power_driver = IPMIPowerDriver() env = select_c_utf8_locale() popen_mock = self.patch(ipmi_module, 'Popen') process = popen_mock.return_value - process.communicate.side_effect = [(b'', b''), (b'off', b'')] + process.communicate.side_effect = [(b'off', b'')] process.returncode = 0 result = ipmi_power_driver._issue_ipmi_command('off', **context) self.expectThat( popen_mock, MockCallsMatch( - call(ipmi_chassis_config_command, stdout=PIPE, - stderr=PIPE, env=env), call(ipmipower_command, stdout=PIPE, stderr=PIPE, env=env))) self.expectThat(result, Equals('off')) @@ -324,3 +309,58 @@ self.assertThat( _issue_ipmi_command_mock, MockCalledOnceWith('query', **context)) + + def test__issue_ipmi_chassis_config_with_power_boot_type(self): + context = make_context() + driver = IPMIPowerDriver() + ip_address = factory.make_ipv4_address() + find_ip_via_arp = self.patch(ipmi_module, 'find_ip_via_arp') + find_ip_via_arp.return_value = ip_address + power_change = "on" + + context['mac_address'] = factory.make_mac_address() + context['power_address'] = random.choice((None, "", " ")) + context['power_boot_type'] = IPMI_BOOT_TYPE.EFI + + self.patch_autospec(driver, "_issue_ipmi_chassis_config_command") + self.patch_autospec(driver, "_issue_ipmipower_command") + driver._issue_ipmi_command(power_change, **context) + + # The IP address is passed to _issue_ipmi_chassis_config_command. + self.assertThat( + driver._issue_ipmi_chassis_config_command, + MockCalledOnceWith( + ANY, power_change, ip_address, + power_boot_type=IPMI_BOOT_TYPE.EFI)) + # The IP address is also within the command passed to + # _issue_ipmi_chassis_config_command. + self.assertThat( + driver._issue_ipmi_chassis_config_command.call_args[0], + Contains(ip_address)) + # The IP address is passed to _issue_ipmipower_command. + self.assertThat( + driver._issue_ipmipower_command, + MockCalledOnceWith(ANY, power_change, ip_address)) + + def test__chassis_config_written_to_temporary_file_with_boot_type(self): + boot_type = self.patch(ipmi_module, "power_boot_type") + boot_type.return_value = IPMI_BOOT_TYPE.EFI + NamedTemporaryFile = self.patch(ipmi_module, "NamedTemporaryFile") + tmpfile = NamedTemporaryFile.return_value + tmpfile.__enter__.return_value = tmpfile + tmpfile.name = factory.make_name("filename") + + IPMIPowerDriver._issue_ipmi_chassis_config_command( + ["true"], sentinel.change, sentinel.addr, + power_boot_type=IPMI_BOOT_TYPE.EFI) + + self.assertThat( + NamedTemporaryFile, MockCalledOnceWith("w+", encoding="utf-8")) + self.assertThat(tmpfile.__enter__, MockCalledOnceWith()) + self.assertThat( + tmpfile.write, + MockCalledOnceWith( + IPMI_CONFIG_WITH_BOOT_TYPE % IPMI_BOOT_TYPE_MAPPING[ + IPMI_BOOT_TYPE.EFI])) + self.assertThat(tmpfile.flush, MockCalledOnceWith()) + self.assertThat(tmpfile.__exit__, MockCalledOnceWith(None, None, None)) diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/boot_resources.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/boot_resources.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/boot_resources.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/boot_resources.py 2018-08-17 02:41:34.000000000 +0000 @@ -9,6 +9,7 @@ ] from argparse import ArgumentParser +import copy import errno from io import StringIO import os @@ -46,10 +47,15 @@ read_text_file, tempdir, ) +from provisioningserver.utils.service_monitor import ServiceActionError from provisioningserver.utils.shell import ( call_and_check, ExternalProcessError, ) +from provisioningserver.utils.snappy import ( + get_snap_data_path, + running_in_snap, +) from twisted.internet.defer import inlineCallbacks from twisted.python.filepath import FilePath @@ -205,7 +211,13 @@ def update_targets_conf(snapshot): """Runs tgt-admin to update the new targets from "maas.tgt".""" # Ensure that tgt is running before tgt-admin is used. - service_monitor.ensureService("tgt").wait(30) + try: + service_monitor.ensureService("tgt").wait(30) + except ServiceActionError: + msg = "Unable to start tgt" + try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg) + maaslog.warning(msg) + return # Update the tgt config. targets_conf = os.path.join(snapshot, 'maas.tgt') @@ -216,11 +228,18 @@ return try: + env = copy.deepcopy(os.environ) + # LP:1718706 - When TGT is run in a snap the socket is stored in the + # SNAP_DATA path. Define it before calling tgt-admin otherwise the + # standard path is used and the call will fail. + if running_in_snap(): + env['TGT_IPC_SOCKET'] = os.path.join( + get_snap_data_path(), 'tgtd-socket') call_and_check(sudo([ get_path('/usr/sbin/tgt-admin'), '--conf', targets_conf, '--update', 'ALL', - ])) + ]), env=env) except ExternalProcessError as e: msg = "Unable to update TGT config: %s" % e try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg) diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/download_resources.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/download_resources.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/download_resources.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/download_resources.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2014-2017 Canonical Ltd. This software is licensed under the +# Copyright 2014-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Simplestreams code to download boot resources.""" @@ -302,8 +302,12 @@ if 'subarch' in item: # MAAS uses the 'generic' subarch when it doesn't know which # subarch to use. This happens during enlistment and commissioning. - # Allow the 'generic' kflavor to own the 'generic' hardlink. - if item.get('kflavor') == 'generic': + # Allow the 'generic' kflavor to own the 'generic' hardlink. The + # generic kernel should always be the ga kernel for xenial+, + # hwe- for older releases. + if (item.get('kflavor') == 'generic' and ( + item['subarch'].startswith('ga-') or + item['subarch'] == 'hwe-%s' % item['release'][0])): subarches = {item['subarch'], 'generic'} else: subarches = {item['subarch']} diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/tests/test_boot_resources.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/tests/test_boot_resources.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/tests/test_boot_resources.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/tests/test_boot_resources.py 2018-08-17 02:41:34.000000000 +0000 @@ -54,6 +54,7 @@ tempdir, write_text_file, ) +from provisioningserver.utils.service_monitor import ServiceActionError from provisioningserver.utils.shell import ExternalProcessError from testtools.content import Content from testtools.content_type import UTF8_TEXT @@ -596,6 +597,18 @@ boot_resources.update_targets_conf(factory.make_name("snapshot")) self.assertThat(mock_ensureService, MockCalledOnceWith("tgt")) + def test_update_targets_conf_logs_tgt_service_check_error(self): + # Regression test for LP:1735025 + mock_ensureService = self.patch( + boot_resources.service_monitor, "ensureService") + mock_ensureService.side_effect = ServiceActionError() + mock_try_send_rack_event = self.patch( + boot_resources, 'try_send_rack_event') + mock_maaslog = self.patch(boot_resources.maaslog, 'warning') + boot_resources.update_targets_conf(factory.make_name("snapshot")) + self.assertThat(mock_try_send_rack_event, MockCalledOnce()) + self.assertThat(mock_maaslog, MockCalledOnce()) + def test_update_targets_conf_logs_error(self): self.patch(boot_resources.service_monitor, "ensureService") mock_try_send_rack_event = self.patch( @@ -614,7 +627,7 @@ MockCalledOnceWith([ 'sudo', '-n', '/usr/sbin/tgt-admin', '--conf', os.path.join(snapshot, 'maas.tgt'), - '--update', 'ALL'])) + '--update', 'ALL'], env=os.environ)) def test_update_targets_only_runs_when_conf_exists(self): # Regression test for LP:1655721 @@ -632,6 +645,29 @@ MockCalledOnceWith(os.path.join(temp_dir, 'maas.tgt'))) self.assertThat(mock_call_and_check, MockNotCalled()) + def test_update_targets_conf_sets_env_var_in_snap(self): + # Regression test for LP:1718706 + self.patch(boot_resources.service_monitor, "ensureService") + self.patch(boot_resources.os.path, 'exists').return_value = True + self.patch(boot_resources, 'call_and_check') + self.patch(boot_resources, 'running_in_snap').return_value = True + snap_data_path = factory.make_name('snap_data_path') + self.patch(boot_resources, 'get_snap_data_path').return_value = ( + snap_data_path) + snapshot = factory.make_name("snapshot") + boot_resources.update_targets_conf(snapshot) + self.assertThat( + boot_resources.call_and_check, + MockCalledOnceWith([ + 'sudo', '-n', '/usr/sbin/tgt-admin', + '--conf', os.path.join(snapshot, 'maas.tgt'), + '--update', 'ALL' + ], env={ + 'TGT_IPC_SOCKET': os.path.join( + snap_data_path, 'tgtd-socket'), + **os.environ, + })) + class TestMetaContains(MAASTestCase): """Tests for the `meta_contains` function.""" diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/tests/test_download_resources.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/tests/test_download_resources.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/import_images/tests/test_download_resources.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/import_images/tests/test_download_resources.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2014-2016 Canonical Ltd. This software is licensed under the +# Copyright 2014-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Tests for `provisioningserver.import_images.download_resources`.""" @@ -361,7 +361,7 @@ label=product['label'], subarches={'ga-16.04'}, bootloader_type=None)) - def test_inserts_generic_link_for_generic_kflavor(self): + def test_inserts_generic_link_for_generic_ga_kflavor(self): product_mapping = ProductMapping() product = self.make_product(subarch='ga-16.04', kflavor='generic') product_mapping.add(product, 'ga-16.04') @@ -391,6 +391,69 @@ label=product['label'], subarches={'ga-16.04', 'generic'}, bootloader_type=None)) + def test_inserts_no_generic_link_for_generic_non_ga_kflavor(self): + # Regression test for LP:1749246 + product_mapping = ProductMapping() + product = self.make_product(subarch='hwe-16.04', kflavor='generic') + product_mapping.add(product, 'hwe-16.04') + repo_writer = download_resources.RepoWriter( + None, None, product_mapping) + self.patch( + download_resources, 'products_exdata').return_value = product + # Prevent MAAS from trying to actually write the file. + mock_insert_file = self.patch(download_resources, 'insert_file') + mock_link_resources = self.patch(download_resources, 'link_resources') + # We only need to provide the product as the other fields are only used + # when writing the actual files to disk. + repo_writer.insert_item(product, None, None, None, None) + # None is used for the store and the content source as we're not + # writing anything to disk. + self.assertThat( + mock_insert_file, + MockCalledOnceWith( + None, os.path.basename(product['path']), product['sha256'], + {'sha256': product['sha256']}, product['size'], None)) + # links are mocked out by the mock_insert_file above. + self.assertThat( + mock_link_resources, + MockCalledOnceWith( + snapshot_path=None, links=mock.ANY, osystem=product['os'], + arch=product['arch'], release=product['release'], + label=product['label'], subarches={'hwe-16.04'}, + bootloader_type=None)) + + def test_inserts_generic_link_for_generic_kflavor_old_hwe_style_ga(self): + # Regression test for LP:1768323 + product_mapping = ProductMapping() + product = self.make_product( + subarch='hwe-p', kflavor='generic', release='precise') + product_mapping.add(product, 'hwe-p') + repo_writer = download_resources.RepoWriter( + None, None, product_mapping) + self.patch( + download_resources, 'products_exdata').return_value = product + # Prevent MAAS from trying to actually write the file. + mock_insert_file = self.patch(download_resources, 'insert_file') + mock_link_resources = self.patch(download_resources, 'link_resources') + # We only need to provide the product as the other fields are only used + # when writing the actual files to disk. + repo_writer.insert_item(product, None, None, None, None) + # None is used for the store and the content source as we're not + # writing anything to disk. + self.assertThat( + mock_insert_file, + MockCalledOnceWith( + None, os.path.basename(product['path']), product['sha256'], + {'sha256': product['sha256']}, product['size'], None)) + # links are mocked out by the mock_insert_file above. + self.assertThat( + mock_link_resources, + MockCalledOnceWith( + snapshot_path=None, links=mock.ANY, osystem=product['os'], + arch=product['arch'], release=product['release'], + label=product['label'], subarches={'hwe-p', 'generic'}, + bootloader_type=None)) + class TestLinkResources(MAASTestCase): """Tests for `LinkResources`().""" diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/maas_api_helper.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/maas_api_helper.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/maas_api_helper.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/maas_api_helper.py 2018-08-17 02:41:34.000000000 +0000 @@ -234,7 +234,11 @@ if None not in (power_type, power_params): params[b'power_type'] = power_type.encode('utf-8') - user, power_pass, power_address, driver = power_params.split(",") + if power_type == 'moonshot': + user, power_pass, power_address, driver = power_params.split(",") + else: + (user, power_pass, power_address, + driver, boot_type) = power_params.split(",") # OrderedDict is used to make testing easier. power_params = OrderedDict([ ('power_user', user), @@ -245,6 +249,7 @@ power_params['power_hwaddress'] = driver else: power_params['power_driver'] = driver + power_params['power_boot_type'] = boot_type params[b'power_parameters'] = json.dumps(power_params).encode() data, headers = encode_multipart_data( diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/node_info_scripts.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/node_info_scripts.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/node_info_scripts.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/node_info_scripts.py 2018-08-17 02:41:34.000000000 +0000 @@ -287,8 +287,8 @@ def get_iface_list(ifconfig_output): return [ - line.split()[0] - for line in ifconfig_output.splitlines()[1:]] + line.split()[1].split(b':')[0].split(b'@')[0] + for line in ifconfig_output.splitlines()] def has_ipv4_address(iface): output = check_output(('ip', '-4', 'addr', 'list', 'dev', iface)) @@ -309,8 +309,9 @@ # a configured ipv6 interface, since ipv6 won't work there. return no_ipv6_found - all_ifaces = get_iface_list(check_output(("ifconfig", "-s", "-a"))) - configured_ifaces = get_iface_list(check_output(("ifconfig", "-s"))) + all_ifaces = get_iface_list(check_output(("ip", "-o", "link", "show"))) + configured_ifaces = get_iface_list(check_output( + ("ip", "-o", "link", "show", "up"))) configured_ifaces_4 = [ iface for iface in configured_ifaces if has_ipv4_address(iface)] configured_ifaces_6 = [ diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/tests/test_maas_api_helper.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/tests/test_maas_api_helper.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/tests/test_maas_api_helper.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/tests/test_maas_api_helper.py 2018-08-17 02:41:34.000000000 +0000 @@ -334,6 +334,7 @@ ('power_pass', factory.make_name('power_pass')), ('power_address', factory.make_url()), ('power_driver', factory.make_name('power_driver')), + ('power_boot_type', factory.make_name('power_boot_type')), ]) # None used for url and creds as we're not actually sending data. diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/tests/test_node_info_scripts.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/tests/test_node_info_scripts.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/refresh/tests/test_node_info_scripts.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/refresh/tests/test_node_info_scripts.py 2018-08-17 02:41:34.000000000 +0000 @@ -209,33 +209,63 @@ # The two following example outputs differ because eth2 and eth1 are not -# configured and thus 'ifconfig -s -a' returns a list with both 'eth1' -# and 'eth2' while 'ifconfig -s' does not contain them. +# configured and thus 'ip -o link show' returns a list with both 'eth1' +# and 'eth2' while 'ip -o link show up' does not contain them. -# Example output of 'ifconfig -s -a': -ifconfig_all = b"""\ -Iface MTU Met RX-OK RX-ERR RX-DRP RX-OVR TX-OK TX-ERR TX-DRP -eth2 1500 0 0 0 0 0 0 0 -eth1 1500 0 0 0 0 0 0 0 -eth0 1500 0 1366127 0 0 0 831110 0 -eth4 1500 0 0 0 0 0 0 0 -eth5 1500 0 0 0 0 0 0 0 -eth6 1500 0 0 0 0 0 0 0 -lo 65536 0 38075 0 0 0 38075 0 -virbr0 1500 0 0 0 0 0 0 0 -wlan0 1500 0 2304695 0 0 0 1436049 0 -""" - -# Example output of 'ifconfig -s': -ifconfig_config = b"""\ -Iface MTU Met RX-OK RX-ERR RX-DRP RX-OVR TX-OK TX-ERR TX-DRP -eth0 1500 0 1366127 0 0 0 831110 0 -eth4 1500 0 1366127 0 0 0 831110 0 -eth5 1500 0 1366127 0 0 0 831110 0 -eth6 1500 0 1366127 0 0 0 831110 0 -lo 65536 0 38115 0 0 0 38115 0 -virbr0 1500 0 0 0 0 0 0 0 -wlan0 1500 0 2304961 0 0 0 1436319 0 +# Example output of 'ip -o link show': +ip_link_show_all = b"""\ +1: eth2: mtu 1500 qdisc noop state DOWN mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:08 brd \ + ff:ff:ff:ff:ff:ff +2: eth1: mtu 1500 qdisc noop state DOWN mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:07 brd \ + ff:ff:ff:ff:ff:ff +3: eth0: mtu 1500 qdisc mq state UP mode \ + DEFAULT group default qlen 1000\\ link/ether 00:01:02:03:04:03 brd \ + ff:ff:ff:ff:ff:ff +4: eth4: mtu 1500 qdisc noop state DOWN mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:04 brd \ + ff:ff:ff:ff:ff:ff +5: eth5: mtu 1500 qdisc noop state DOWN mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:06 brd \ + ff:ff:ff:ff:ff:ff +6: eth6: mtu 1500 qdisc noop state DOWN mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:06 brd \ + ff:ff:ff:ff:ff:ff +7: lo: mtu 65536 qdisc noqueue state UNKNOWN mode \ + DEFAULT group default qlen 1000\\ link/loopback 00:00:00:00:00:00 brd \ + 00:00:00:00:00:00 +8: virbr0: mtu 1500 qdisc noop state DOWN mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:02 brd \ + ff:ff:ff:ff:ff:ff +9: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:05 brd \ + ff:ff:ff:ff:ff:ff +""" + +# Example output of 'ip -o link show up': +ip_link_show = b"""\ +1: eth0: mtu 1500 qdisc mq state UP mode \ + DEFAULT group default qlen 1000\\ link/ether 00:01:02:03:04:03 brd \ + ff:ff:ff:ff:ff:ff +2: eth4: mtu 1500 qdisc noop state UP mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:04 brd \ + ff:ff:ff:ff:ff:ff +3: eth5: mtu 1500 qdisc noop state UP mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:06 brd \ + ff:ff:ff:ff:ff:ff +4: eth6: mtu 1500 qdisc noop state UP mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:06 brd \ + ff:ff:ff:ff:ff:ff +5: lo: mtu 65536 qdisc noqueue state UNKNOWN mode \ + DEFAULT group default qlen 1000\\ link/loopback 00:00:00:00:00:00 \ + brd 00:00:00:00:00:00 +6: virbr0: mtu 1500 qdisc noop state UP mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:02 brd \ + ff:ff:ff:ff:ff:ff +7: wlan0: mtu 1500 qdisc noop state UP mode DEFAULT \ + group default qlen 1000\\ link/ether 00:01:02:03:04:05 brd \ + ff:ff:ff:ff:ff:ff """ # Example output of 'ip addr list dev XX': @@ -306,7 +336,7 @@ def test_calls_dhclient_on_unconfigured_interfaces(self): check_output = self.patch(subprocess, "check_output") check_output.side_effect = [ - ifconfig_all, ifconfig_config, + ip_link_show_all, ip_link_show, ip_eth0, ip_eth4, ip_eth5, ip_eth6, ip_lo, ip_virbr0, ip_wlan0, ip_eth0, ip_eth4, ip_eth5, ip_eth6, ip_lo, ip_virbr0, ip_wlan0 ] diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/dhcp.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/dhcp.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/dhcp.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/dhcp.py 2018-08-17 02:41:34.000000000 +0000 @@ -154,10 +154,12 @@ dhcpd_config, interfaces_config = state.get_config(server) try: sudo_write_file( - server.config_filename, dhcpd_config.encode("utf-8")) + server.config_filename, dhcpd_config.encode("utf-8"), + mode=0o640) sudo_write_file( server.interfaces_filename, - interfaces_config.encode("utf-8")) + interfaces_config.encode("utf-8"), + mode=0o640) except ExternalProcessError as e: # ExternalProcessError.__str__ contains a generic failure message # as well as the command and its error output. On the other hand, diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/power.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/power.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/power.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/power.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2014-2016 Canonical Ltd. This software is licensed under the +# Copyright 2014-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Power control.""" @@ -285,6 +285,9 @@ system_id, hostname, power_type, power_change, context) if power_type not in PowerDriverRegistry: returnValue(None) + power_driver = PowerDriverRegistry.get_item(power_type) + if not power_driver.queryable: + returnValue(None) new_power_state = yield perform_power_driver_query( system_id, hostname, power_type, context) if new_power_state == "unknown" or new_power_state == power_change: diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/tests/test_dhcp.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/tests/test_dhcp.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/tests/test_dhcp.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/tests/test_dhcp.py 2018-08-17 02:41:34.000000000 +0000 @@ -574,10 +574,12 @@ MockCallsMatch( call( self.server.config_filename, - expected_config.encode("utf-8")), + expected_config.encode("utf-8"), + mode=0o640), call( self.server.interfaces_filename, - interface["name"].encode("utf-8")), + interface["name"].encode("utf-8"), + mode=0o640), )) self.assertThat(on, MockCalledOnceWith()) self.assertThat( @@ -623,10 +625,12 @@ MockCallsMatch( call( self.server.config_filename, - expected_config.encode("utf-8")), + expected_config.encode("utf-8"), + mode=0o640), call( self.server.interfaces_filename, - interface["name"].encode("utf-8")), + interface["name"].encode("utf-8"), + mode=0o640), )) self.assertThat(on, MockCalledOnceWith()) self.assertThat( @@ -673,10 +677,12 @@ MockCallsMatch( call( self.server.config_filename, - expected_config.encode("utf-8")), + expected_config.encode("utf-8"), + mode=0o640), call( self.server.interfaces_filename, - interface["name"].encode("utf-8")), + interface["name"].encode("utf-8"), + mode=0o640), )) self.assertThat(on, MockCalledOnceWith()) self.assertThat( @@ -730,10 +736,12 @@ MockCallsMatch( call( self.server.config_filename, - expected_config.encode("utf-8")), + expected_config.encode("utf-8"), + mode=0o640), call( self.server.interfaces_filename, - interface["name"].encode("utf-8")), + interface["name"].encode("utf-8"), + mode=0o640), )) self.assertThat(on, MockCalledOnceWith()) self.assertThat( @@ -801,10 +809,12 @@ MockCallsMatch( call( self.server.config_filename, - expected_config.encode("utf-8")), + expected_config.encode("utf-8"), + mode=0o640), call( self.server.interfaces_filename, - interface["name"].encode("utf-8")), + interface["name"].encode("utf-8"), + mode=0o640), )) self.assertThat(on, MockCalledOnceWith()) self.assertThat( @@ -876,10 +886,12 @@ MockCallsMatch( call( self.server.config_filename, - expected_config.encode("utf-8")), + expected_config.encode("utf-8"), + mode=0o640), call( self.server.interfaces_filename, - interface["name"].encode("utf-8")), + interface["name"].encode("utf-8"), + mode=0o640), )) self.assertThat(on, MockCalledOnceWith()) self.assertThat( diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/tests/test_power.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/tests/test_power.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/rpc/tests/test_power.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/rpc/tests/test_power.py 2018-08-17 02:41:34.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2014-2016 Canonical Ltd. This software is licensed under the +# Copyright 2014-2018 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Tests for :py:module:`~provisioningserver.rpc.power`.""" @@ -10,6 +10,7 @@ from unittest.mock import ( ANY, call, + MagicMock, sentinel, ) @@ -19,6 +20,7 @@ MockCalledOnceWith, MockCalledWith, MockCallsMatch, + MockNotCalled, ) from maastesting.testcase import ( MAASTestCase, @@ -244,6 +246,37 @@ system_id, hostname, power_change)) @inlineCallbacks + def test__return_none_when_unqueryable(self): + system_id = factory.make_name('system_id') + hostname = factory.make_name('hostname') + power_driver = random.choice([ + driver + for _, driver in PowerDriverRegistry + if not driver.queryable + ]) + power_change = 'on' + context = { + factory.make_name('context-key'): factory.make_name('context-val') + } + self.patch(power, 'is_driver_available').return_value = True + get_item = self.patch(PowerDriverRegistry, 'get_item') + get_item.return_value = MagicMock() + get_item.return_value.queryable = False + perform_power_driver_query = self.patch( + power, 'perform_power_driver_query') + perform_power_driver_query.return_value = succeed(power_change) + self.patch(power, 'power_change_success') + yield self.patch_rpc_methods() + + result = yield power.change_power_state( + system_id, hostname, power_driver.name, power_change, context) + + self.expectThat(get_item, MockCalledWith(power_driver.name)) + self.expectThat(perform_power_driver_query, MockNotCalled()) + self.expectThat(power.power_change_success, MockNotCalled()) + self.expectThat(result, Equals(None)) + + @inlineCallbacks def test__calls_power_driver_on_for_power_driver(self): system_id = factory.make_name('system_id') hostname = factory.make_name('hostname') @@ -267,7 +300,7 @@ result = yield power.change_power_state( system_id, hostname, power_driver.name, power_change, context) - self.expectThat(get_item, MockCalledOnceWith(power_driver.name)) + self.expectThat(get_item, MockCalledWith(power_driver.name)) self.expectThat( perform_power_driver_query, MockCalledOnceWith( system_id, hostname, power_driver.name, context)) @@ -300,7 +333,7 @@ result = yield power.change_power_state( system_id, hostname, power_driver.name, power_change, context) - self.expectThat(get_item, MockCalledOnceWith(power_driver.name)) + self.expectThat(get_item, MockCalledWith(power_driver.name)) self.expectThat( perform_power_driver_query, MockCalledOnceWith( system_id, hostname, power_driver.name, context)) @@ -333,7 +366,7 @@ result = yield power.change_power_state( system_id, hostname, power_driver.name, power_change, context) - self.expectThat(get_item, MockCalledOnceWith(power_driver.name)) + self.expectThat(get_item, MockCalledWith(power_driver.name)) self.expectThat( perform_power_driver_query, MockCalledOnceWith( system_id, hostname, power_driver.name, context)) diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/templates/proxy/maas-proxy.conf.template maas-2.3.5-6511-gf466fdb/src/provisioningserver/templates/proxy/maas-proxy.conf.template --- maas-2.3.0-6434-gd354690/src/provisioningserver/templates/proxy/maas-proxy.conf.template 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/templates/proxy/maas-proxy.conf.template 2018-08-17 02:41:34.000000000 +0000 @@ -23,7 +23,11 @@ http_access allow localhost http_access deny all http_port 3128 transparent +{{if not maas_proxy_port}} http_port 8000 +{{else}} +http_port {{maas_proxy_port}} +{{endif}} refresh_pattern ^ftp: 1440 20% 10080 refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/templates/uefi/config.local.amd64.template maas-2.3.5-6511-gf466fdb/src/provisioningserver/templates/uefi/config.local.amd64.template --- maas-2.3.0-6434-gd354690/src/provisioningserver/templates/uefi/config.local.amd64.template 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/templates/uefi/config.local.amd64.template 2018-08-17 02:41:34.000000000 +0000 @@ -6,8 +6,14 @@ {{if kernel_params.osystem == "windows"}} search --set=root --file /efi/Microsoft/Boot/bootmgfw.efi chainloader /efi/Microsoft/Boot/bootmgfw.efi + {{elif kernel_params.osystem == "centos"}} + search --set=root --file /efi/centos/grubx64.efi + chainloader /efi/centos/grubx64.efi + {{elif kernel_params.osystem == "rhel"}} + search --set=root --file /efi/redhat/grubx64.efi + chainloader /efi/redhat/grubx64.efi {{else}} - search --set=root --file /efi/ubuntu/shimx64.efi - chainloader /efi/ubuntu/shimx64.efi + search --set=root --file /efi/ubuntu/grubx64.efi + chainloader /efi/ubuntu/grubx64.efi {{endif}} } diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/utils/__init__.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/__init__.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/utils/__init__.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/__init__.py 2018-08-17 02:41:34.000000000 +0000 @@ -16,7 +16,10 @@ ] from collections import Iterable -from functools import reduce +from functools import ( + lru_cache, + reduce, +) from itertools import chain import os from pipes import quote @@ -60,6 +63,13 @@ '..', 'templates', *path)) +@lru_cache(256) +def load_template(*path: Tuple[str]): + """Load the template.""" + return tempita.Template.from_filename( + locate_template(*path), encoding="UTF-8") + + def dict_depth(d, depth=0): """Returns the max depth of a dictionary.""" if not isinstance(d, dict) or not d: diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/utils/netplan.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/netplan.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/utils/netplan.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/netplan.py 2018-08-17 02:41:34.000000000 +0000 @@ -55,11 +55,12 @@ "bond-active-slave": None, "bond-fail-over-mac": None, "bond-master": None, - "bond-num-unsol-na": None, # XXX: use bond-num-grat-arp? "bond-primary": None, "bond-queue-id": None, "bond-slaves": None, "bond-use-carrier": None, + # This is just an internal alias for bond-num-grat-arp. + "bond-num-unsol-na": "gratuitous-arp", } diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/utils/network.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/network.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/utils/network.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/network.py 2018-08-17 02:41:34.000000000 +0000 @@ -946,6 +946,9 @@ # the fact that netaddr assumes all the data is ASCII, sometimes # netaddr will raise an exception during this process. return None + except IndexError: + # See bug #1748031; this is another way netaddr can fail. + return None except NotRegisteredError: # This could happen for locally-administered MACs. return None diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/utils/tests/test_network.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/tests/test_network.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/utils/tests/test_network.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/tests/test_network.py 2018-08-17 02:41:34.000000000 +0000 @@ -83,7 +83,6 @@ resolves_to_loopback_address, reverseResolve, ) -from provisioningserver.utils.shell import call_and_check from testtools import ExpectedException from testtools.matchers import ( Contains, @@ -107,13 +106,6 @@ ) -installed_curtin_version = call_and_check([ - "dpkg-query", "--showformat=${Version}", - "--show", "python3-curtin"]).decode("ascii") -installed_curtin_version = int( - installed_curtin_version.split("~bzr", 1)[1].split("-", 1)[0]) - - class TestMakeNetwork(MAASTestCase): def test_constructs_IPNetwork(self): @@ -246,6 +238,14 @@ organization = get_eui_organization(mock_eui) self.assertThat(organization, Is(None)) + def test_get_eui_organization_returns_None_for_IndexError(self): + mock_eui = Mock() + mock_eui.oui = Mock() + mock_eui.oui.registration = Mock() + mock_eui.oui.registration.side_effect = IndexError + organization = get_eui_organization(mock_eui) + self.assertThat(organization, Is(None)) + def test_get_eui_organization_returns_none_for_invalid_mac(self): organization = get_eui_organization(EUI("FF:FF:b7:00:00:00")) self.assertThat(organization, Is(None)) diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/utils/tests/test_version.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/tests/test_version.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/utils/tests/test_version.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/tests/test_version.py 2018-08-17 02:41:34.000000000 +0000 @@ -392,11 +392,17 @@ }), ("stable", { "version": "2.3.0-6192-g10a4565-0ubuntu1", - "output": ("2.3/stable"), + # XXX - revert this once snap store easily supports tracks + # for MAAS snap. + # "output": ("2.3/stable"), + "output": ("latest/stable"), }), ("stable", { "version": "", - "output": ("%s/stable" % '.'.join(old_version.split('.')[0:2])), + # XXX - revert this once snap store easily supports tracks + # for MAAS snap. + # "output": ("%s/stable" % '.'.join(old_version.split('.')[0:2])), + "output": ("latest/stable"), }), ] diff -Nru maas-2.3.0-6434-gd354690/src/provisioningserver/utils/version.py maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/version.py --- maas-2.3.0-6434-gd354690/src/provisioningserver/utils/version.py 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/src/provisioningserver/utils/version.py 2018-08-17 02:41:34.000000000 +0000 @@ -107,7 +107,13 @@ if 'alpha' in series or 'beta' in series or 'rc' in series: return "latest/edge" else: - return "%s/stable" % '.'.join(series.split('.')[0:2]) + # XXX - The snap store doesn't make it easy to create tracks + # for projects. As such, MAAS will continue to use the latest + # stable channel for the snap publishing regardless of the + # version. As such, use the stable channel instead. This needs + # to be reverted once tracks are easily supported by the store. + # return "%s/stable" % '.'.join(series.split('.')[0:2]) + return "latest/stable" MAASVersion = namedtuple('MAASVersion', ( diff -Nru maas-2.3.0-6434-gd354690/versions.cfg maas-2.3.5-6511-gf466fdb/versions.cfg --- maas-2.3.0-6434-gd354690/versions.cfg 2017-11-20 18:58:10.000000000 +0000 +++ maas-2.3.5-6511-gf466fdb/versions.cfg 2018-08-17 02:41:34.000000000 +0000 @@ -42,7 +42,7 @@ traitlets = 4.3.1 unittest2 = 1.1.0 wcwidth = 0.1.7 -zc.buildout = 2.9.5 +zc.buildout = 2.10.0 zc.recipe.egg = 2.0.3 # Lint.