diff -Nru cloud-init-19.2-36-g059d049c/ChangeLog cloud-init-19.3-41-gc4735dd3/ChangeLog --- cloud-init-19.2-36-g059d049c/ChangeLog 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/ChangeLog 2019-12-03 21:45:14.000000000 +0000 @@ -1,3 +1,102 @@ +19.3 + - azure: support matching dhcp route-metrics for dual-stack ipv4 ipv6 + (LP: #1850308) + - configdrive: fix subplatform config-drive for /config-drive source + [David Kindred] (LP: #1849731) + - DataSourceSmartOS: reconfigure network on each boot + [Mike Gerdts] (LP: #1765801) + - Add config for ssh-key import and consuming user-data [Pavel Zakharov] + - net: fix subnet_is_ipv6() for stateless|stateful + [Harald Jensås] (LP: #1848690) + - OVF: disable custom script execution by default [Xiaofeng Wang] + - cc_puppet: Implement csr_attributes.yaml support [Matthias Baur] + - cloud-init.service: on centos/fedora/redhat wait on NetworkManager.service + (LP: #1843334) + - azure: Do not lock user on instance id change [Sam Eiderman] (LP: #1849677) + - net/netplan: use ipv6-mtu key for specifying ipv6 mtu values + - Fix usages of yaml, and move yaml_dump to safeyaml.dumps. (LP: #1849640) + - exoscale: Increase url_max_wait to 120s. [Chris Glass] + - net/sysconfig: fix available check on SUSE distros + [Robert Schweikert] (LP: #1849378) + - docs: Fix incorrect Azure IMDS IP address [Joshua Powers] (LP: #1849508) + - introduce .travis.yml + - net: enable infiniband support in eni and sysconfig renderers + [Darren Birkett] (LP: #1847114) + - guestcust_util: handle special characters in config file [Xiaofeng Wang] + - fix some more typos in comments [Dominic Schlegel] + - replace any deprecated log.warn with log.warning + [Dominic Schlegel] (LP: #1508442) + - net: handle openstack dhcpv6-stateless configuration + [Harald Jensås] (LP: #1847517) + - Add .venv/ to .gitignore [Dominic Schlegel] + - Small typo fixes in code comments. [Dominic Schlegel] + - cloud_test/lxd: Retry container delete a few times + - Add Support for e24cloud to Ec2 datasource. (LP: #1696476) + - Add RbxCloud datasource [Adam Dobrawy] + - get_interfaces: don't exclude bridge and bond members (LP: #1846535) + - Add support for Arch Linux in render-cloudcfg [Conrad Hoffmann] + - util: json.dumps on python 2.7 will handle UnicodeDecodeError on binary + (LP: #1801364) + - debian/ubuntu: add missing word to netplan/ENI header (LP: #1845669) + - ovf: do not generate random instance-id for IMC customization path + - sysconfig: only write resolv.conf if network_state has DNS values + (LP: #1843634) + - sysconfig: use distro variant to check if available (LP: #1843584) + - systemd/cloud-init.service.tmpl: start after wicked.service + [Robert Schweikert] + - docs: fix zstack documentation lints + - analyze/show: remove trailing space in output + - Add missing space in warning: "not avalid seed" [Brian Candler] + - pylintrc: add 'enter_context' to generated-members list + - Add datasource for ZStack platform. [Shixin Ruan] (LP: #1841181) + - docs: organize TOC and update summary of project [Joshua Powers] + - tools: make clean now cleans the dev directory, not the system + - docs: create cli specific page [Joshua Powers] + - docs: added output examples to analyze.rst [Joshua Powers] + - docs: doc8 fixes for instancedata page [Joshua Powers] + - docs: clean up formatting, organize boot page [Joshua Powers] + - net: add is_master check for filtering device list (LP: #1844191) + - docs: more complete list of availability [Joshua Powers] + - docs: start FAQ page [Joshua Powers] + - docs: cleanup output & order of datasource page [Joshua Powers] + - Brightbox: restrict detection to require full domain match .brightbox.com + - VMWware: add option into VMTools config to enable/disable custom script. + [Xiaofeng Wang] + - net,Oracle: Add support for netfailover detection + - atomic_helper: add DEBUG logging to write_file (LP: #1843276) + - doc: document doc, create makefile and tox target [Joshua Powers] + - .gitignore: ignore files produced by package builds + - docs: fix whitespace, spelling, and line length [Joshua Powers] + - docs: remove unnecessary file in doc directory [Joshua Powers] + - Oracle: Render secondary vnic IP and MTU values only + - exoscale: fix sysconfig cloud_config_modules overrides (LP: #1841454) + - net/cmdline: refactor to allow multiple initramfs network config sources + - ubuntu-drivers: call db_x_loadtemplatefile to accept NVIDIA EULA + (LP: #1840080) + - Add missing #cloud-config comment on first example in documentation. + [Florian Müller] + - ubuntu-drivers: emit latelink=true debconf to accept nvidia eula + (LP: #1840080) + - DataSourceOracle: prefer DS network config over initramfs + - format.rst: add text/jinja2 to list of content types (+ cleanups) + - Add GitHub pull request template to point people at hacking doc + - cloudinit/distros/parsers/sys_conf: add docstring to SysConf + - pyflakes: remove unused variable [Joshua Powers] + - Azure: Record boot timestamps, system information, and diagnostic events + [Anh Vo] + - DataSourceOracle: configure secondary NICs on Virtual Machines + - distros: fix confusing variable names + - azure/net: generate_fallback_nic emits network v2 config instead of v1 + - Add support for publishing host keys to GCE guest attributes [Rick Wright] + - New data source for the Exoscale.com cloud platform [Chris Glass] + - doc: remove intersphinx extension + - cc_set_passwords: rewrite documentation (LP: #1838794) + - net/cmdline: split interfaces_by_mac and init network config determination + - stages: allow data sources to override network config source order + - cloud_tests: updates and fixes + - Fix bug rendering MTU on bond or vlan when input was netplan. (LP: #1836949) + - net: update net sequence, include wait on netdevs, opensuse netrules path + (LP: #1817368) 19.2: - net: add rfc3442 (classless static routes) to EphemeralDHCP (LP: #1821102) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/analyze/show.py cloud-init-19.3-41-gc4735dd3/cloudinit/analyze/show.py --- cloud-init-19.2-36-g059d049c/cloudinit/analyze/show.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/analyze/show.py 2019-12-03 21:45:14.000000000 +0000 @@ -349,7 +349,7 @@ if event_name(event) == event_name(prev_evt): record = event_record(start_time, prev_evt, event) records.append(format_record("Finished stage: " - "(%n) %d seconds ", + "(%n) %d seconds", record) + "\n") total_time += record.get('delta') else: diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/apport.py cloud-init-19.3-41-gc4735dd3/cloudinit/apport.py --- cloud-init-19.2-36-g059d049c/cloudinit/apport.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/apport.py 2019-12-03 21:45:14.000000000 +0000 @@ -22,6 +22,7 @@ 'CloudSigma', 'CloudStack', 'DigitalOcean', + 'E24Cloud', 'GCE - Google Compute Engine', 'Exoscale', 'Hetzner Cloud', @@ -37,6 +38,7 @@ 'Scaleway', 'SmartOS', 'VMware', + 'ZStack', 'Other'] # Potentially clear text collected logs diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/cmd/devel/net_convert.py cloud-init-19.3-41-gc4735dd3/cloudinit/cmd/devel/net_convert.py --- cloud-init-19.2-36-g059d049c/cloudinit/cmd/devel/net_convert.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/cmd/devel/net_convert.py 2019-12-03 21:45:14.000000000 +0000 @@ -5,13 +5,12 @@ import json import os import sys -import yaml from cloudinit.sources.helpers import openstack from cloudinit.sources import DataSourceAzure as azure from cloudinit.sources import DataSourceOVF as ovf -from cloudinit import distros +from cloudinit import distros, safeyaml from cloudinit.net import eni, netplan, network_state, sysconfig from cloudinit import log @@ -78,13 +77,12 @@ if args.kind == "eni": pre_ns = eni.convert_eni_data(net_data) elif args.kind == "yaml": - pre_ns = yaml.load(net_data) + pre_ns = safeyaml.load(net_data) if 'network' in pre_ns: pre_ns = pre_ns.get('network') if args.debug: sys.stderr.write('\n'.join( - ["Input YAML", - yaml.dump(pre_ns, default_flow_style=False, indent=4), ""])) + ["Input YAML", safeyaml.dumps(pre_ns), ""])) elif args.kind == 'network_data.json': pre_ns = openstack.convert_net_json( json.loads(net_data), known_macs=known_macs) @@ -100,9 +98,8 @@ "input data") if args.debug: - sys.stderr.write('\n'.join([ - "", "Internal State", - yaml.dump(ns, default_flow_style=False, indent=4), ""])) + sys.stderr.write('\n'.join( + ["", "Internal State", safeyaml.dumps(ns), ""])) distro_cls = distros.fetch(args.distro) distro = distro_cls(args.distro, {}, None) config = {} @@ -116,6 +113,8 @@ config['postcmds'] = False # trim leading slash config['netplan_path'] = config['netplan_path'][1:] + # enable some netplan features + config['features'] = ['dhcp-use-domains', 'ipv6-mtu'] else: r_cls = sysconfig.Renderer config = distro.renderer_configs.get('sysconfig') diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/cmd/tests/test_main.py cloud-init-19.3-41-gc4735dd3/cloudinit/cmd/tests/test_main.py --- cloud-init-19.2-36-g059d049c/cloudinit/cmd/tests/test_main.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/cmd/tests/test_main.py 2019-12-03 21:45:14.000000000 +0000 @@ -6,8 +6,9 @@ from six import StringIO from cloudinit.cmd import main +from cloudinit import safeyaml from cloudinit.util import ( - ensure_dir, load_file, write_file, yaml_dumps) + ensure_dir, load_file, write_file) from cloudinit.tests.helpers import ( FilesystemMockingTestCase, wrap_and_call) @@ -39,7 +40,7 @@ ], 'cloud_init_modules': ['write-files', 'runcmd'], } - cloud_cfg = yaml_dumps(self.cfg) + cloud_cfg = safeyaml.dumps(self.cfg) ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) self.cloud_cfg_file = os.path.join( self.new_root, 'etc', 'cloud', 'cloud.cfg') @@ -113,7 +114,7 @@ """When local-hostname metadata is present, call cc_set_hostname.""" self.cfg['datasource'] = { 'None': {'metadata': {'local-hostname': 'md-hostname'}}} - cloud_cfg = yaml_dumps(self.cfg) + cloud_cfg = safeyaml.dumps(self.cfg) write_file(self.cloud_cfg_file, cloud_cfg) cmdargs = myargs( debug=False, files=None, force=False, local=False, reporter=None, diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_apt_pipelining.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_apt_pipelining.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_apt_pipelining.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_apt_pipelining.py 2019-12-03 21:45:14.000000000 +0000 @@ -59,7 +59,7 @@ elif apt_pipe_value_s in [str(b) for b in range(0, 6)]: write_apt_snippet(apt_pipe_value_s, log, DEFAULT_FILE) else: - log.warn("Invalid option for apt_pipelining: %s", apt_pipe_value) + log.warning("Invalid option for apt_pipelining: %s", apt_pipe_value) def write_apt_snippet(setting, log, f_name): diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_byobu.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_byobu.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_byobu.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_byobu.py 2019-12-03 21:45:14.000000000 +0000 @@ -60,7 +60,7 @@ valid = ("enable-user", "enable-system", "enable", "disable-user", "disable-system", "disable") if value not in valid: - log.warn("Unknown value %s for byobu_by_default", value) + log.warning("Unknown value %s for byobu_by_default", value) mod_user = value.endswith("-user") mod_sys = value.endswith("-system") @@ -80,8 +80,8 @@ (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) (user, _user_config) = ug_util.extract_default(users) if not user: - log.warn(("No default byobu user provided, " - "can not launch %s for the default user"), bl_inst) + log.warning(("No default byobu user provided, " + "can not launch %s for the default user"), bl_inst) else: shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) shcmd += " || X=$(($X+1)); " diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_chef.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_chef.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_chef.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_chef.py 2019-12-03 21:45:14.000000000 +0000 @@ -196,7 +196,7 @@ # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: log.debug(("Skipping module named %s," - " no 'chef' key in configuration"), name) + " no 'chef' key in configuration"), name) return chef_cfg = cfg['chef'] @@ -215,9 +215,9 @@ if vcert != "system": util.write_file(vkey_path, vcert) elif not os.path.isfile(vkey_path): - log.warn("chef validation_cert provided as 'system', but " - "validation_key path '%s' does not exist.", - vkey_path) + log.warning("chef validation_cert provided as 'system', but " + "validation_key path '%s' does not exist.", + vkey_path) # Create the chef config from template template_fn = cloud.get_template_filename('chef_client.rb') @@ -234,8 +234,8 @@ util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: - log.warn("No template found, not rendering to %s", - CHEF_RB_PATH) + log.warning("No template found, not rendering to %s", + CHEF_RB_PATH) # Set the firstboot json fb_filename = util.get_cfg_option_str(chef_cfg, 'firstboot_path', @@ -276,9 +276,9 @@ elif isinstance(cmd_args, six.string_types): cmd.append(cmd_args) else: - log.warn("Unknown type %s provided for chef" - " 'exec_arguments' expected list, tuple," - " or string", type(cmd_args)) + log.warning("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) cmd.extend(CHEF_EXEC_DEF_ARGS) else: cmd.extend(CHEF_EXEC_DEF_ARGS) @@ -334,7 +334,7 @@ retries=util.get_cfg_option_int(chef_cfg, "omnibus_url_retries"), omnibus_version=omnibus_version) else: - log.warn("Unknown chef install type '%s'", install_type) + log.warning("Unknown chef install type '%s'", install_type) run = False return run diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_debug.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_debug.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_debug.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_debug.py 2019-12-03 21:45:14.000000000 +0000 @@ -33,6 +33,7 @@ from cloudinit import type_utils from cloudinit import util +from cloudinit import safeyaml SKIP_KEYS = frozenset(['log_cfgs']) @@ -49,7 +50,7 @@ def _dumps(obj): - text = util.yaml_dumps(obj, explicit_start=False, explicit_end=False) + text = safeyaml.dumps(obj, explicit_start=False, explicit_end=False) return text.rstrip() diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_emit_upstart.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_emit_upstart.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_emit_upstart.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_emit_upstart.py 2019-12-03 21:45:14.000000000 +0000 @@ -69,6 +69,6 @@ util.subp(cmd) except Exception as e: # TODO(harlowja), use log exception from utils?? - log.warn("Emission of upstart event %s failed due to: %s", n, e) + log.warning("Emission of upstart event %s failed due to: %s", n, e) # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_final_message.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_final_message.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_final_message.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_final_message.py 2019-12-03 21:45:14.000000000 +0000 @@ -83,6 +83,6 @@ util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) if cloud.datasource.is_disconnected: - log.warn("Used fallback datasource") + log.warning("Used fallback datasource") # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_growpart.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_growpart.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_growpart.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_growpart.py 2019-12-03 21:45:14.000000000 +0000 @@ -321,7 +321,7 @@ mycfg = cfg.get('growpart') if not isinstance(mycfg, dict): - log.warn("'growpart' in config was not a dict") + log.warning("'growpart' in config was not a dict") return mode = mycfg.get('mode', "auto") diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_keys_to_console.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_keys_to_console.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_keys_to_console.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_keys_to_console.py 2019-12-03 21:45:14.000000000 +0000 @@ -52,8 +52,8 @@ def handle(name, cfg, cloud, log, _args): helper_path = _get_helper_tool_path(cloud.distro) if not os.path.exists(helper_path): - log.warn(("Unable to activate module %s," - " helper tool not found at %s"), name, helper_path) + log.warning(("Unable to activate module %s," + " helper tool not found at %s"), name, helper_path) return fp_blacklist = util.get_cfg_option_list(cfg, @@ -68,7 +68,7 @@ util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) except Exception: - log.warn("Writing keys to the system console failed!") + log.warning("Writing keys to the system console failed!") raise # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_lxd.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_lxd.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_lxd.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_lxd.py 2019-12-03 21:45:14.000000000 +0000 @@ -66,21 +66,21 @@ name) return if not isinstance(lxd_cfg, dict): - log.warn("lxd config must be a dictionary. found a '%s'", - type(lxd_cfg)) + log.warning("lxd config must be a dictionary. found a '%s'", + type(lxd_cfg)) return # Grab the configuration init_cfg = lxd_cfg.get('init') if not isinstance(init_cfg, dict): - log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + log.warning("lxd/init config must be a dictionary. found a '%s'", + type(init_cfg)) init_cfg = {} bridge_cfg = lxd_cfg.get('bridge', {}) if not isinstance(bridge_cfg, dict): - log.warn("lxd/bridge config must be a dictionary. found a '%s'", - type(bridge_cfg)) + log.warning("lxd/bridge config must be a dictionary. found a '%s'", + type(bridge_cfg)) bridge_cfg = {} # Install the needed packages @@ -95,7 +95,7 @@ try: cloud.distro.install_packages(packages) except util.ProcessExecutionError as exc: - log.warn("failed to install packages %s: %s", packages, exc) + log.warning("failed to install packages %s: %s", packages, exc) return # Set up lxd if init config is given @@ -301,5 +301,4 @@ raise e LOG.debug(msg, nic_name, profile, fail_assume_enoent) - # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_mounts.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_mounts.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_mounts.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_mounts.py 2019-12-03 21:45:14.000000000 +0000 @@ -251,10 +251,10 @@ util.ensure_dir(tdir) util.log_time(LOG.debug, msg, func=util.subp, args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - '{ fallocate -l "${2}M" "$1" || ' - ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + ('rm -f "$1" && umask 0066 && ' + '{ fallocate -l "${2}M" "$1" || ' + 'dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), 'setup_swap', fname, mbsize]]) except Exception as e: @@ -347,8 +347,8 @@ for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): - log.warn("Mount option %s not a list, got a %s instead", - (i + 1), type_utils.obj_name(cfgmnt[i])) + log.warning("Mount option %s not a list, got a %s instead", + (i + 1), type_utils.obj_name(cfgmnt[i])) continue start = str(cfgmnt[i][0]) @@ -495,7 +495,7 @@ util.subp(cmd) log.debug(fmt, "PASS") except util.ProcessExecutionError: - log.warn(fmt, "FAIL") + log.warning(fmt, "FAIL") util.logexc(log, fmt, "FAIL") # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_package_update_upgrade_install.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_package_update_upgrade_install.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_package_update_upgrade_install.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_package_update_upgrade_install.py 2019-12-03 21:45:14.000000000 +0000 @@ -108,7 +108,8 @@ reboot_fn_exists = os.path.isfile(REBOOT_FILE) if (upgrade or pkglist) and reboot_if_required and reboot_fn_exists: try: - log.warn("Rebooting after upgrade or install per %s", REBOOT_FILE) + log.warning("Rebooting after upgrade or install per " + "%s", REBOOT_FILE) # Flush the above warning + anything else out... logging.flushLoggers(log) _fire_reboot(log) @@ -117,8 +118,8 @@ errors.append(e) if len(errors): - log.warn("%s failed with exceptions, re-raising the last one", - len(errors)) + log.warning("%s failed with exceptions, re-raising the last one", + len(errors)) raise errors[-1] # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_phone_home.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_phone_home.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_phone_home.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_phone_home.py 2019-12-03 21:45:14.000000000 +0000 @@ -79,8 +79,8 @@ ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: - log.warn(("Skipping module named %s, " - "no 'url' found in 'phone_home' configuration"), name) + log.warning(("Skipping module named %s, " + "no 'url' found in 'phone_home' configuration"), name) return url = ph_cfg['url'] @@ -91,7 +91,7 @@ except Exception: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " - "using %s instead", tries) + "using %s instead", tries) if post_list == "all": post_list = POST_LIST_ALL @@ -112,7 +112,7 @@ all_keys[n] = util.load_file(path) except Exception: util.logexc(log, "%s: failed to open, can not phone home that " - "data!", path) + "data!", path) submit_keys = {} for k in post_list: @@ -120,8 +120,8 @@ submit_keys[k] = all_keys[k] else: submit_keys[k] = None - log.warn(("Requested key %s from 'post'" - " configuration list not available"), k) + log.warning(("Requested key %s from 'post'" + " configuration list not available"), k) # Get them read to be posted real_submit_keys = {} diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_power_state_change.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_power_state_change.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_power_state_change.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_power_state_change.py 2019-12-03 21:45:14.000000000 +0000 @@ -103,24 +103,23 @@ return False else: if log: - log.warn(pre + "unexpected exit %s. " % ret + - "do not apply change.") + log.warning(pre + "unexpected exit %s. " % ret + + "do not apply change.") return False except Exception as e: if log: - log.warn(pre + "Unexpected error: %s" % e) + log.warning(pre + "Unexpected error: %s" % e) return False def handle(_name, cfg, _cloud, log, _args): - try: (args, timeout, condition) = load_power_state(cfg) if args is None: log.debug("no power_state provided. doing nothing") return except Exception as e: - log.warn("%s Not performing power state change!" % str(e)) + log.warning("%s Not performing power state change!" % str(e)) return if condition is False: @@ -131,7 +130,7 @@ cmdline = givecmdline(mypid) if not cmdline: - log.warn("power_state: failed to get cmdline of current process") + log.warning("power_state: failed to get cmdline of current process") return devnull_fp = open(os.devnull, "w") @@ -214,7 +213,7 @@ def fatal(msg): if log: - log.warn(msg) + log.warning(msg) doexit(EXIT_FAIL) known_errnos = (errno.ENOENT, errno.ESRCH) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_puppet.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_puppet.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_puppet.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_puppet.py 2019-12-03 21:45:14.000000000 +0000 @@ -24,9 +24,10 @@ The module also provides keys for configuring the new puppet 4 paths and installing the puppet package from the puppetlabs repositories: https://docs.puppet.com/puppet/4.2/reference/whered_it_go.html -The keys are ``package_name``, ``conf_file`` and ``ssl_dir``. If unset, their -values will default to ones that work with puppet 3.x and with distributions -that ship modified puppet 4.x that uses the old paths. +The keys are ``package_name``, ``conf_file``, ``ssl_dir`` and +``csr_attributes_path``. If unset, their values will default to +ones that work with puppet 3.x and with distributions that ship modified +puppet 4.x that uses the old paths. Puppet configuration can be specified under the ``conf`` key. The configuration is specified as a dictionary containing high-level ``
`` @@ -40,6 +41,10 @@ instead will be used as the puppermaster certificate. It should be specified in pem format as a multi-line string (using the ``|`` yaml notation). +Additionally it's possible to create a csr_attributes.yaml for +CSR attributes and certificate extension requests. +See https://puppet.com/docs/puppet/latest/config_file_csr_attributes.html + **Internal name:** ``cc_puppet`` **Module frequency:** per instance @@ -53,6 +58,7 @@ version: conf_file: '/etc/puppet/puppet.conf' ssl_dir: '/var/lib/puppet/ssl' + csr_attributes_path: '/etc/puppet/csr_attributes.yaml' package_name: 'puppet' conf: agent: @@ -62,28 +68,39 @@ -------BEGIN CERTIFICATE------- -------END CERTIFICATE------- + csr_attributes: + custom_attributes: + 1.2.840.113549.1.9.7: 342thbjkt82094y0uthhor289jnqthpc2290 + extension_requests: + pp_uuid: ED803750-E3C7-44F5-BB08-41A04433FE2E + pp_image_name: my_ami_image + pp_preshared_key: 342thbjkt82094y0uthhor289jnqthpc2290 """ from six import StringIO import os import socket +import yaml from cloudinit import helpers from cloudinit import util PUPPET_CONF_PATH = '/etc/puppet/puppet.conf' PUPPET_SSL_DIR = '/var/lib/puppet/ssl' +PUPPET_CSR_ATTRIBUTES_PATH = '/etc/puppet/csr_attributes.yaml' PUPPET_PACKAGE_NAME = 'puppet' class PuppetConstants(object): - def __init__(self, puppet_conf_file, puppet_ssl_dir, log): + def __init__(self, puppet_conf_file, puppet_ssl_dir, + csr_attributes_path, log): self.conf_path = puppet_conf_file self.ssl_dir = puppet_ssl_dir self.ssl_cert_dir = os.path.join(puppet_ssl_dir, "certs") self.ssl_cert_path = os.path.join(self.ssl_cert_dir, "ca.pem") + self.csr_attributes_path = csr_attributes_path def _autostart_puppet(log): @@ -98,8 +115,8 @@ elif os.path.exists('/sbin/chkconfig'): util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: - log.warn(("Sorry we do not know how to enable" - " puppet services on this system")) + log.warning(("Sorry we do not know how to enable" + " puppet services on this system")) def handle(name, cfg, cloud, log, _args): @@ -118,11 +135,13 @@ conf_file = util.get_cfg_option_str( puppet_cfg, 'conf_file', PUPPET_CONF_PATH) ssl_dir = util.get_cfg_option_str(puppet_cfg, 'ssl_dir', PUPPET_SSL_DIR) + csr_attributes_path = util.get_cfg_option_str( + puppet_cfg, 'csr_attributes_path', PUPPET_CSR_ATTRIBUTES_PATH) - p_constants = PuppetConstants(conf_file, ssl_dir, log) + p_constants = PuppetConstants(conf_file, ssl_dir, csr_attributes_path, log) if not install and version: - log.warn(("Puppet install set false but version supplied," - " doing nothing.")) + log.warning(("Puppet install set false but version supplied," + " doing nothing.")) elif install: log.debug(("Attempting to install puppet %s,"), version if version else 'latest') @@ -141,7 +160,7 @@ cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = '\n'.join(cleaned_lines) # Move to puppet_config.read_file when dropping py2.7 - puppet_config.readfp( # pylint: disable=W1505 + puppet_config.readfp( # pylint: disable=W1505 StringIO(cleaned_contents), filename=p_constants.conf_path) for (cfg_name, cfg) in puppet_cfg['conf'].items(): @@ -176,6 +195,11 @@ % (p_constants.conf_path)) util.write_file(p_constants.conf_path, puppet_config.stringify()) + if 'csr_attributes' in puppet_cfg: + util.write_file(p_constants.csr_attributes_path, + yaml.dump(puppet_cfg['csr_attributes'], + default_flow_style=False)) + # Set it up so it autostarts _autostart_puppet(log) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_resizefs.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_resizefs.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_resizefs.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_resizefs.py 2019-12-03 21:45:14.000000000 +0000 @@ -8,7 +8,6 @@ """Resizefs: cloud-config module which resizes the filesystem""" - import errno import getopt import os @@ -183,7 +182,7 @@ not container): devpath = util.rootdev_from_cmdline(util.get_cmdline()) if devpath is None: - log.warn("Unable to find device '/dev/root'") + log.warning("Unable to find device '/dev/root'") return None log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) @@ -212,8 +211,8 @@ log.debug("Device '%s' did not exist in container. " "cannot resize: %s", devpath, info) elif exc.errno == errno.ENOENT: - log.warn("Device '%s' did not exist. cannot resize: %s", - devpath, info) + log.warning("Device '%s' did not exist. cannot resize: %s", + devpath, info) else: raise exc return None @@ -223,8 +222,8 @@ log.debug("device '%s' not a block device in container." " cannot resize: %s" % (devpath, info)) else: - log.warn("device '%s' not a block device. cannot resize: %s" % - (devpath, info)) + log.warning("device '%s' not a block device. cannot resize: %s" % + (devpath, info)) return None return devpath # The writable block devpath @@ -243,7 +242,7 @@ resize_what = "/" result = util.get_mount_info(resize_what, log) if not result: - log.warn("Could not determine filesystem type of %s", resize_what) + log.warning("Could not determine filesystem type of %s", resize_what) return (devpth, fs_type, mount_point) = result @@ -280,8 +279,8 @@ break if not resizer: - log.warn("Not resizing unknown filesystem type %s for %s", - fs_type, resize_what) + log.warning("Not resizing unknown filesystem type %s for %s", + fs_type, resize_what) return resize_cmd = resizer(resize_what, devpth) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_resolv_conf.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_resolv_conf.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_resolv_conf.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_resolv_conf.py 2019-12-03 21:45:14.000000000 +0000 @@ -102,11 +102,11 @@ return if "resolv_conf" not in cfg: - log.warn("manage_resolv_conf True but no parameters provided!") + log.warning("manage_resolv_conf True but no parameters provided!") template_fn = cloud.get_template_filename('resolv.conf') if not template_fn: - log.warn("No template found, not rendering /etc/resolv.conf") + log.warning("No template found, not rendering /etc/resolv.conf") return generate_resolv_conf(template_fn=template_fn, params=cfg["resolv_conf"]) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_rightscale_userdata.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_rightscale_userdata.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_rightscale_userdata.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_rightscale_userdata.py 2019-12-03 21:45:14.000000000 +0000 @@ -111,8 +111,8 @@ log.debug("%s urls were skipped or failed", skipped) if captured_excps: - log.warn("%s failed with exceptions, re-raising the last one", - len(captured_excps)) + log.warning("%s failed with exceptions, re-raising the last one", + len(captured_excps)) raise captured_excps[-1] # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_rsyslog.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_rsyslog.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_rsyslog.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_rsyslog.py 2019-12-03 21:45:14.000000000 +0000 @@ -432,7 +432,7 @@ systemd=cloud.distro.uses_systemd()), except util.ProcessExecutionError as e: restarted = False - log.warn("Failed to reload syslog", e) + log.warning("Failed to reload syslog", e) if restarted: # This only needs to run if we *actually* restarted diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_salt_minion.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_salt_minion.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_salt_minion.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_salt_minion.py 2019-12-03 21:45:14.000000000 +0000 @@ -45,7 +45,7 @@ import os -from cloudinit import util +from cloudinit import safeyaml, util # Note: see https://docs.saltstack.com/en/latest/topics/installation/ # Note: see https://docs.saltstack.com/en/latest/ref/configuration/ @@ -59,7 +59,7 @@ # constants tailored for FreeBSD if util.is_FreeBSD(): - self.pkg_name = 'py27-salt' + self.pkg_name = 'py36-salt' self.srv_name = 'salt_minion' self.conf_dir = '/usr/local/etc/salt' # constants for any other OS @@ -97,13 +97,13 @@ if 'conf' in s_cfg: # Add all sections from the conf object to minion config file minion_config = os.path.join(const.conf_dir, 'minion') - minion_data = util.yaml_dumps(s_cfg.get('conf')) + minion_data = safeyaml.dumps(s_cfg.get('conf')) util.write_file(minion_config, minion_data) if 'grains' in s_cfg: # add grains to /etc/salt/grains grains_config = os.path.join(const.conf_dir, 'grains') - grains_data = util.yaml_dumps(s_cfg.get('grains')) + grains_data = safeyaml.dumps(s_cfg.get('grains')) util.write_file(grains_config, grains_data) # ... copy the key pair if specified diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_per_boot.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_per_boot.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_per_boot.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_per_boot.py 2019-12-03 21:45:14.000000000 +0000 @@ -40,8 +40,8 @@ try: util.runparts(runparts_path) except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning("Failed to run module %s (%s in %s)", + name, SCRIPT_SUBDIR, runparts_path) raise # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_per_instance.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_per_instance.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_per_instance.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_per_instance.py 2019-12-03 21:45:14.000000000 +0000 @@ -40,8 +40,8 @@ try: util.runparts(runparts_path) except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning("Failed to run module %s (%s in %s)", + name, SCRIPT_SUBDIR, runparts_path) raise # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_per_once.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_per_once.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_per_once.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_per_once.py 2019-12-03 21:45:14.000000000 +0000 @@ -40,8 +40,8 @@ try: util.runparts(runparts_path) except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning("Failed to run module %s (%s in %s)", + name, SCRIPT_SUBDIR, runparts_path) raise # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_user.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_user.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_user.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_user.py 2019-12-03 21:45:14.000000000 +0000 @@ -44,8 +44,8 @@ try: util.runparts(runparts_path) except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning("Failed to run module %s (%s in %s)", + name, SCRIPT_SUBDIR, runparts_path) raise # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_vendor.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_vendor.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_scripts_vendor.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_scripts_vendor.py 2019-12-03 21:45:14.000000000 +0000 @@ -48,8 +48,8 @@ try: util.runparts(runparts_path, exe_prefix=prefix) except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning("Failed to run module %s (%s in %s)", + name, SCRIPT_SUBDIR, runparts_path) raise # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_seed_random.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_seed_random.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_seed_random.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_seed_random.py 2019-12-03 21:45:14.000000000 +0000 @@ -131,7 +131,7 @@ env['RANDOM_SEED_FILE'] = seed_path handle_random_seed_command(command=command, required=req, env=env) except ValueError as e: - log.warn("handling random command [%s] failed: %s", command, e) + log.warning("handling random command [%s] failed: %s", command, e) raise e # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_set_hostname.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_set_hostname.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_set_hostname.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_set_hostname.py 2019-12-03 21:45:14.000000000 +0000 @@ -21,7 +21,9 @@ the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set, ``fqdn`` will be used. -**Internal name:** per instance +**Internal name:** ``cc_set_hostname`` + +**Module frequency:** per instance **Supported distros:** all diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_set_passwords.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_set_passwords.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_set_passwords.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_set_passwords.py 2019-12-03 21:45:14.000000000 +0000 @@ -164,7 +164,7 @@ if user: plist = ["%s:%s" % (user, password)] else: - log.warn("No default or defined user to change password for.") + log.warning("No default or defined user to change password for.") errors = [] if plist: @@ -179,20 +179,21 @@ for line in plist: u, p = line.split(':', 1) if prog.match(p) is not None and ":" not in p: - hashed_plist_in.append("%s:%s" % (u, p)) + hashed_plist_in.append(line) hashed_users.append(u) else: + # in this else branch, we potentially change the password + # hence, a deviation from .append(line) if p == "R" or p == "RANDOM": p = rand_user_password() randlist.append("%s:%s" % (u, p)) plist_in.append("%s:%s" % (u, p)) users.append(u) - ch_in = '\n'.join(plist_in) + '\n' if users: try: log.debug("Changing password for %s:", users) - util.subp(['chpasswd'], ch_in) + chpasswd(cloud.distro, ch_in) except Exception as e: errors.append(e) util.logexc( @@ -202,7 +203,7 @@ if hashed_users: try: log.debug("Setting hashed password for %s:", hashed_users) - util.subp(['chpasswd', '-e'], hashed_ch_in) + chpasswd(cloud.distro, hashed_ch_in, hashed=True) except Exception as e: errors.append(e) util.logexc( @@ -218,7 +219,7 @@ expired_users = [] for u in users: try: - util.subp(['passwd', '--expire', u]) + cloud.distro.expire_passwd(u) expired_users.append(u) except Exception as e: errors.append(e) @@ -238,4 +239,14 @@ def rand_user_password(pwlen=9): return util.rand_str(pwlen, select_from=PW_SET) + +def chpasswd(distro, plist_in, hashed=False): + if util.is_FreeBSD(): + for pentry in plist_in.splitlines(): + u, p = pentry.split(":") + distro.set_passwd(u, p, hashed=hashed) + else: + cmd = ['chpasswd'] + (['-e'] if hashed else []) + util.subp(cmd, plist_in) + # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_snappy.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_snappy.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_snappy.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_snappy.py 2019-12-03 21:45:14.000000000 +0000 @@ -68,6 +68,7 @@ from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import temp_utils +from cloudinit import safeyaml from cloudinit import util import glob @@ -188,7 +189,7 @@ # Note, however, we do not touch config files on disk. nested_cfg = {'config': {shortname: config}} (fd, cfg_tmpf) = temp_utils.mkstemp() - os.write(fd, util.yaml_dumps(nested_cfg).encode()) + os.write(fd, safeyaml.dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_ssh.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_ssh.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_ssh.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_ssh.py 2019-12-03 21:45:14.000000000 +0000 @@ -56,9 +56,13 @@ no-port-forwarding,no-agent-forwarding,no-X11-forwarding Authorized keys for the default user/first user defined in ``users`` can be -specified using `ssh_authorized_keys``. Keys should be specified as a list of +specified using ``ssh_authorized_keys``. Keys should be specified as a list of public keys. +Importing ssh public keys for the default user (defined in ``users``)) is +enabled by default. This feature may be disabled by setting +``allow_publish_ssh_keys: false``. + .. note:: see the ``cc_set_passwords`` module documentation to enable/disable ssh password authentication @@ -91,6 +95,7 @@ ssh_authorized_keys: - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEA3FSyQwBI6Z+nCSjUU ... - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZ ... + allow_public_ssh_keys: ssh_publish_hostkeys: enabled: (Defaults to true) blacklist: (Defaults to [dsa]) @@ -207,7 +212,13 @@ disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", ssh_util.DISABLE_USER_OPTS) - keys = cloud.get_public_ssh_keys() or [] + keys = [] + if util.get_cfg_option_bool(cfg, 'allow_public_ssh_keys', True): + keys = cloud.get_public_ssh_keys() or [] + else: + log.debug('Skipping import of publish ssh keys per ' + 'config setting: allow_public_ssh_keys=False') + if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_update_etc_hosts.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_update_etc_hosts.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_update_etc_hosts.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_update_etc_hosts.py 2019-12-03 21:45:14.000000000 +0000 @@ -62,8 +62,8 @@ if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: - log.warn(("Option 'manage_etc_hosts' was set," - " but no hostname was found")) + log.warning(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return # Render from a template file @@ -80,8 +80,8 @@ elif manage_hosts == "localhost": (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: - log.warn(("Option 'manage_etc_hosts' was set," - " but no hostname was found")) + log.warning(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return log.debug("Managing localhost in /etc/hosts") diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/cc_yum_add_repo.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_yum_add_repo.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/cc_yum_add_repo.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/cc_yum_add_repo.py 2019-12-03 21:45:14.000000000 +0000 @@ -113,16 +113,16 @@ missing_required = 0 for req_field in ['baseurl']: if req_field not in repo_config: - log.warn(("Repository %s does not contain a %s" - " configuration 'required' entry"), - repo_id, req_field) + log.warning(("Repository %s does not contain a %s" + " configuration 'required' entry"), + repo_id, req_field) missing_required += 1 if not missing_required: repo_configs[canon_repo_id] = repo_config repo_locations[canon_repo_id] = repo_fn_pth else: - log.warn("Repository %s is missing %s required fields, skipping!", - repo_id, missing_required) + log.warning("Repository %s is missing %s required fields, " + "skipping!", repo_id, missing_required) for (c_repo_id, path) in repo_locations.items(): repo_blob = _format_repository_config(c_repo_id, repo_configs.get(c_repo_id)) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/tests/test_set_passwords.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/tests/test_set_passwords.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/tests/test_set_passwords.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/tests/test_set_passwords.py 2019-12-03 21:45:14.000000000 +0000 @@ -74,7 +74,7 @@ with_logs = True - def test_handle_on_empty_config(self): + def test_handle_on_empty_config(self, *args): """handle logs that no password has changed when config is empty.""" cloud = self.tmp_cloud(distro='ubuntu') setpass.handle( @@ -108,4 +108,44 @@ '\n'.join(valid_hashed_pwds) + '\n')], m_subp.call_args_list) + @mock.patch(MODPATH + "util.is_FreeBSD") + @mock.patch(MODPATH + "util.subp") + def test_freebsd_calls_custom_pw_cmds_to_set_and_expire_passwords( + self, m_subp, m_is_freebsd): + """FreeBSD calls custom pw commands instead of chpasswd and passwd""" + m_is_freebsd.return_value = True + cloud = self.tmp_cloud(distro='freebsd') + valid_pwds = ['ubuntu:passw0rd'] + cfg = {'chpasswd': {'list': valid_pwds}} + setpass.handle( + 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) + self.assertEqual([ + mock.call(['pw', 'usermod', 'ubuntu', '-h', '0'], data='passw0rd', + logstring="chpasswd for ubuntu"), + mock.call(['pw', 'usermod', 'ubuntu', '-p', '01-Jan-1970'])], + m_subp.call_args_list) + + @mock.patch(MODPATH + "util.is_FreeBSD") + @mock.patch(MODPATH + "util.subp") + def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp, + m_is_freebsd): + """handle parses command set random passwords.""" + m_is_freebsd.return_value = False + cloud = self.tmp_cloud(distro='ubuntu') + valid_random_pwds = [ + 'root:R', + 'ubuntu:RANDOM'] + cfg = {'chpasswd': {'expire': 'false', 'list': valid_random_pwds}} + with mock.patch(MODPATH + 'util.subp') as m_subp: + setpass.handle( + 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) + self.assertIn( + 'DEBUG: Handling input for chpasswd as list.', + self.logs.getvalue()) + self.assertNotEqual( + [mock.call(['chpasswd'], + '\n'.join(valid_random_pwds) + '\n')], + m_subp.call_args_list) + + # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/config/tests/test_ssh.py cloud-init-19.3-41-gc4735dd3/cloudinit/config/tests/test_ssh.py --- cloud-init-19.2-36-g059d049c/cloudinit/config/tests/test_ssh.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/config/tests/test_ssh.py 2019-12-03 21:45:14.000000000 +0000 @@ -5,6 +5,9 @@ from cloudinit.config import cc_ssh from cloudinit import ssh_util from cloudinit.tests.helpers import CiTestCase, mock +import logging + +LOG = logging.getLogger(__name__) MODPATH = "cloudinit.config.cc_ssh." @@ -87,7 +90,7 @@ cc_ssh.PUBLISH_HOST_KEYS = False cloud = self.tmp_cloud( distro='ubuntu', metadata={'public-keys': keys}) - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) options = ssh_util.DISABLE_USER_OPTS.replace("$USER", "NONE") options = options.replace("$DISABLE_USER", "root") m_glob.assert_called_once_with('/etc/ssh/ssh_host_*key*') @@ -103,6 +106,31 @@ @mock.patch(MODPATH + "glob.glob") @mock.patch(MODPATH + "ug_util.normalize_users_groups") @mock.patch(MODPATH + "os.path.exists") + def test_dont_allow_public_ssh_keys(self, m_path_exists, m_nug, + m_glob, m_setup_keys): + """Test allow_public_ssh_keys=False ignores ssh public keys from + platform. + """ + cfg = {"allow_public_ssh_keys": False} + keys = ["key1"] + user = "clouduser" + m_glob.return_value = [] # Return no matching keys to prevent removal + # Mock os.path.exits to True to short-circuit the key writing logic + m_path_exists.return_value = True + m_nug.return_value = ({user: {"default": user}}, {}) + cloud = self.tmp_cloud( + distro='ubuntu', metadata={'public-keys': keys}) + cc_ssh.handle("name", cfg, cloud, LOG, None) + + options = ssh_util.DISABLE_USER_OPTS.replace("$USER", user) + options = options.replace("$DISABLE_USER", "root") + self.assertEqual([mock.call(set(), user), + mock.call(set(), "root", options=options)], + m_setup_keys.call_args_list) + + @mock.patch(MODPATH + "glob.glob") + @mock.patch(MODPATH + "ug_util.normalize_users_groups") + @mock.patch(MODPATH + "os.path.exists") def test_handle_no_cfg_and_default_root(self, m_path_exists, m_nug, m_glob, m_setup_keys): """Test handle with no config and a default distro user.""" @@ -115,7 +143,7 @@ m_nug.return_value = ({user: {"default": user}}, {}) cloud = self.tmp_cloud( distro='ubuntu', metadata={'public-keys': keys}) - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) options = ssh_util.DISABLE_USER_OPTS.replace("$USER", user) options = options.replace("$DISABLE_USER", "root") @@ -140,7 +168,7 @@ m_nug.return_value = ({user: {"default": user}}, {}) cloud = self.tmp_cloud( distro='ubuntu', metadata={'public-keys': keys}) - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) options = ssh_util.DISABLE_USER_OPTS.replace("$USER", user) options = options.replace("$DISABLE_USER", "root") @@ -165,7 +193,7 @@ cloud = self.tmp_cloud( distro='ubuntu', metadata={'public-keys': keys}) cloud.get_public_ssh_keys = mock.Mock(return_value=keys) - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(set(keys), user), mock.call(set(keys), "root", options="")], @@ -196,7 +224,7 @@ cfg = {} expected_call = [self.test_hostkeys[key_type] for key_type in ['ecdsa', 'ed25519', 'rsa']] - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -225,7 +253,7 @@ cfg = {'ssh_publish_hostkeys': {'enabled': True}} expected_call = [self.test_hostkeys[key_type] for key_type in ['ecdsa', 'ed25519', 'rsa']] - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -252,7 +280,7 @@ cloud.datasource.publish_host_keys = mock.Mock() cfg = {'ssh_publish_hostkeys': {'enabled': False}} - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertFalse(cloud.datasource.publish_host_keys.call_args_list) cloud.datasource.publish_host_keys.assert_not_called() @@ -282,7 +310,7 @@ 'blacklist': ['dsa', 'rsa']}} expected_call = [self.test_hostkeys[key_type] for key_type in ['ecdsa', 'ed25519']] - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -312,6 +340,6 @@ 'blacklist': []}} expected_call = [self.test_hostkeys[key_type] for key_type in ['dsa', 'ecdsa', 'ed25519', 'rsa']] - cc_ssh.handle("name", cfg, cloud, None, None) + cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/distros/debian.py cloud-init-19.3-41-gc4735dd3/cloudinit/distros/debian.py --- cloud-init-19.2-36-g059d049c/cloudinit/distros/debian.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/distros/debian.py 2019-12-03 21:45:14.000000000 +0000 @@ -29,9 +29,10 @@ 'enabled': 'auto', } -ENI_HEADER = """# This file is generated from information provided by -# the datasource. Changes to it will not persist across an instance. -# To disable cloud-init's network configuration capabilities, write a file +NETWORK_FILE_HEADER = """\ +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: # network: {config: disabled} """ @@ -48,9 +49,9 @@ } renderer_configs = { "eni": {"eni_path": network_conf_fn["eni"], - "eni_header": ENI_HEADER}, + "eni_header": NETWORK_FILE_HEADER}, "netplan": {"netplan_path": network_conf_fn["netplan"], - "netplan_header": ENI_HEADER, + "netplan_header": NETWORK_FILE_HEADER, "postcmds": True} } diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/distros/freebsd.py cloud-init-19.3-41-gc4735dd3/cloudinit/distros/freebsd.py --- cloud-init-19.2-36-g059d049c/cloudinit/distros/freebsd.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/distros/freebsd.py 2019-12-03 21:45:14.000000000 +0000 @@ -25,6 +25,7 @@ class Distro(distros.Distro): + usr_lib_exec = '/usr/local/lib' rc_conf_fn = "/etc/rc.conf" login_conf_fn = '/etc/login.conf' login_conf_fn_bak = '/etc/login.conf.orig' @@ -233,6 +234,13 @@ if passwd_val is not None: self.set_passwd(name, passwd_val, hashed=True) + def expire_passwd(self, user): + try: + util.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) + except Exception as e: + util.logexc(LOG, "Failed to set pw expiration for %s", user) + raise e + def set_passwd(self, user, passwd, hashed=False): if hashed: hash_opt = "-H" diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/distros/__init__.py cloud-init-19.3-41-gc4735dd3/cloudinit/distros/__init__.py --- cloud-init-19.2-36-g059d049c/cloudinit/distros/__init__.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/distros/__init__.py 2019-12-03 21:45:14.000000000 +0000 @@ -591,6 +591,13 @@ util.logexc(LOG, 'Failed to disable password for user %s', name) raise e + def expire_passwd(self, user): + try: + util.subp(['passwd', '--expire', user]) + except Exception as e: + util.logexc(LOG, "Failed to set 'expire' for %s", user) + raise e + def set_passwd(self, user, passwd, hashed=False): pass_string = '%s:%s' % (user, passwd) cmd = ['chpasswd'] diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/distros/ubuntu.py cloud-init-19.3-41-gc4735dd3/cloudinit/distros/ubuntu.py --- cloud-init-19.2-36-g059d049c/cloudinit/distros/ubuntu.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/distros/ubuntu.py 2019-12-03 21:45:14.000000000 +0000 @@ -30,9 +30,9 @@ } self.renderer_configs = { "eni": {"eni_path": self.network_conf_fn["eni"], - "eni_header": debian.ENI_HEADER}, + "eni_header": debian.NETWORK_FILE_HEADER}, "netplan": {"netplan_path": self.network_conf_fn["netplan"], - "netplan_header": debian.ENI_HEADER, + "netplan_header": debian.NETWORK_FILE_HEADER, "postcmds": True} } diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/ec2_utils.py cloud-init-19.3-41-gc4735dd3/cloudinit/ec2_utils.py --- cloud-init-19.2-36-g059d049c/cloudinit/ec2_utils.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/ec2_utils.py 2019-12-03 21:45:14.000000000 +0000 @@ -134,25 +134,28 @@ return joined -def _skip_retry_on_codes(status_codes, _request_args, cause): +def skip_retry_on_codes(status_codes, _request_args, cause): """Returns False if cause.code is in status_codes.""" return cause.code not in status_codes def get_instance_userdata(api_version='latest', metadata_address='http://169.254.169.254', - ssl_details=None, timeout=5, retries=5): + ssl_details=None, timeout=5, retries=5, + headers_cb=None, exception_cb=None): ud_url = url_helper.combine_url(metadata_address, api_version) ud_url = url_helper.combine_url(ud_url, 'user-data') user_data = '' try: - # It is ok for userdata to not exist (thats why we are stopping if - # NOT_FOUND occurs) and just in that case returning an empty string. - exception_cb = functools.partial(_skip_retry_on_codes, - SKIP_USERDATA_CODES) + if not exception_cb: + # It is ok for userdata to not exist (thats why we are stopping if + # NOT_FOUND occurs) and just in that case returning an empty + # string. + exception_cb = functools.partial(skip_retry_on_codes, + SKIP_USERDATA_CODES) response = url_helper.read_file_or_url( ud_url, ssl_details=ssl_details, timeout=timeout, - retries=retries, exception_cb=exception_cb) + retries=retries, exception_cb=exception_cb, headers_cb=headers_cb) user_data = response.contents except url_helper.UrlError as e: if e.code not in SKIP_USERDATA_CODES: @@ -165,11 +168,13 @@ def _get_instance_metadata(tree, api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5, - leaf_decoder=None): + leaf_decoder=None, headers_cb=None, + exception_cb=None): md_url = url_helper.combine_url(metadata_address, api_version, tree) caller = functools.partial( url_helper.read_file_or_url, ssl_details=ssl_details, - timeout=timeout, retries=retries) + timeout=timeout, retries=retries, headers_cb=headers_cb, + exception_cb=exception_cb) def mcaller(url): return caller(url).contents @@ -191,22 +196,28 @@ def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5, - leaf_decoder=None): + leaf_decoder=None, headers_cb=None, + exception_cb=None): # Note, 'meta-data' explicitly has trailing /. # this is required for CloudStack (LP: #1356855) return _get_instance_metadata(tree='meta-data/', api_version=api_version, metadata_address=metadata_address, ssl_details=ssl_details, timeout=timeout, - retries=retries, leaf_decoder=leaf_decoder) + retries=retries, leaf_decoder=leaf_decoder, + headers_cb=headers_cb, + exception_cb=exception_cb) def get_instance_identity(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5, - leaf_decoder=None): + leaf_decoder=None, headers_cb=None, + exception_cb=None): return _get_instance_metadata(tree='dynamic/instance-identity', api_version=api_version, metadata_address=metadata_address, ssl_details=ssl_details, timeout=timeout, - retries=retries, leaf_decoder=leaf_decoder) + retries=retries, leaf_decoder=leaf_decoder, + headers_cb=headers_cb, + exception_cb=exception_cb) # vi: ts=4 expandtab diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/handlers/cloud_config.py cloud-init-19.3-41-gc4735dd3/cloudinit/handlers/cloud_config.py --- cloud-init-19.2-36-g059d049c/cloudinit/handlers/cloud_config.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/handlers/cloud_config.py 2019-12-03 21:45:14.000000000 +0000 @@ -14,6 +14,7 @@ from cloudinit import log as logging from cloudinit import mergers from cloudinit import util +from cloudinit import safeyaml from cloudinit.settings import (PER_ALWAYS) @@ -75,7 +76,7 @@ '', ] lines.extend(file_lines) - lines.append(util.yaml_dumps(self.cloud_buf)) + lines.append(safeyaml.dumps(self.cloud_buf)) else: lines = [] util.write_file(self.cloud_fn, "\n".join(lines), 0o600) diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/net/eni.py cloud-init-19.3-41-gc4735dd3/cloudinit/net/eni.py --- cloud-init-19.2-36-g059d049c/cloudinit/net/eni.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/net/eni.py 2019-12-03 21:45:14.000000000 +0000 @@ -94,7 +94,7 @@ ] renames = {'mac_address': 'hwaddress'} - if iface['type'] not in ['bond', 'bridge', 'vlan']: + if iface['type'] not in ['bond', 'bridge', 'infiniband', 'vlan']: ignore_map.append('mac_address') for key, value in iface.items(): @@ -399,6 +399,7 @@ def _render_iface(self, iface, render_hwaddress=False): sections = [] subnets = iface.get('subnets', {}) + accept_ra = iface.pop('accept-ra', None) if subnets: for index, subnet in enumerate(subnets): ipv4_subnet_mtu = None @@ -411,8 +412,27 @@ else: ipv4_subnet_mtu = subnet.get('mtu') iface['inet'] = subnet_inet - if subnet['type'].startswith('dhcp'): + if (subnet['type'] == 'dhcp4' or subnet['type'] == 'dhcp6' or + subnet['type'] == 'ipv6_dhcpv6-stateful'): + # Configure network settings using DHCP or DHCPv6 iface['mode'] = 'dhcp' + if accept_ra is not None: + # Accept router advertisements (0=off, 1=on) + iface['accept_ra'] = '1' if accept_ra else '0' + elif subnet['type'] == 'ipv6_dhcpv6-stateless': + # Configure network settings using SLAAC from RAs + iface['mode'] = 'auto' + # Use stateless DHCPv6 (0=off, 1=on) + iface['dhcp'] = '1' + elif subnet['type'] == 'ipv6_slaac': + # Configure network settings using SLAAC from RAs + iface['mode'] = 'auto' + # Use stateless DHCPv6 (0=off, 1=on) + iface['dhcp'] = '0' + elif subnet_is_ipv6(subnet) and subnet['type'] == 'static': + if accept_ra is not None: + # Accept router advertisements (0=off, 1=on) + iface['accept_ra'] = '1' if accept_ra else '0' # do not emit multiple 'auto $IFACE' lines as older (precise) # ifupdown complains @@ -467,9 +487,10 @@ order = { 'loopback': 0, 'physical': 1, - 'bond': 2, - 'bridge': 3, - 'vlan': 4, + 'infiniband': 2, + 'bond': 3, + 'bridge': 4, + 'vlan': 5, } sections = [] diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/net/__init__.py cloud-init-19.3-41-gc4735dd3/cloudinit/net/__init__.py --- cloud-init-19.2-36-g059d049c/cloudinit/net/__init__.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/net/__init__.py 2019-12-03 21:45:14.000000000 +0000 @@ -109,8 +109,22 @@ return os.path.exists(sys_dev_path(devname, "bonding")) -def has_master(devname): - return os.path.exists(sys_dev_path(devname, path="master")) +def get_master(devname): + """Return the master path for devname, or None if no master""" + path = sys_dev_path(devname, path="master") + if os.path.exists(path): + return path + return None + + +def master_is_bridge_or_bond(devname): + """Return a bool indicating if devname's master is a bridge or bond""" + master_path = get_master(devname) + if master_path is None: + return False + bonding_path = os.path.join(master_path, "bonding") + bridge_path = os.path.join(master_path, "bridge") + return (os.path.exists(bonding_path) or os.path.exists(bridge_path)) def is_netfailover(devname, driver=None): @@ -158,7 +172,7 @@ Return True if all of the above is True. """ - if has_master(devname): + if get_master(devname) is not None: return False if driver is None: @@ -215,7 +229,7 @@ Return True if all of the above is True. """ - if not has_master(devname): + if get_master(devname) is None: return False if driver is None: @@ -375,7 +389,7 @@ potential_interfaces = possibly_connected # if eth0 exists use it above anything else, otherwise get the interface - # that we can read 'first' (using the sorted defintion of first). + # that we can read 'first' (using the sorted definition of first). names = list(sorted(potential_interfaces, key=natural_sort_key)) if DEFAULT_PRIMARY_INTERFACE in names: names.remove(DEFAULT_PRIMARY_INTERFACE) @@ -790,7 +804,7 @@ continue if is_bond(name): continue - if has_master(name): + if get_master(name) is not None and not master_is_bridge_or_bond(name): continue if is_netfailover(name): continue diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/net/netplan.py cloud-init-19.3-41-gc4735dd3/cloudinit/net/netplan.py --- cloud-init-19.2-36-g059d049c/cloudinit/net/netplan.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/net/netplan.py 2019-12-03 21:45:14.000000000 +0000 @@ -4,10 +4,11 @@ import os from . import renderer -from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2 +from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2, IPV6_DYNAMIC_TYPES from cloudinit import log as logging from cloudinit import util +from cloudinit import safeyaml from cloudinit.net import SYS_CLASS_NET, get_devicelist KNOWN_SNAPD_CONFIG = b"""\ @@ -34,7 +35,7 @@ if key.startswith(match)) -def _extract_addresses(config, entry, ifname): +def _extract_addresses(config, entry, ifname, features=None): """This method parse a cloudinit.net.network_state dictionary (config) and maps netstate keys/values into a dictionary (entry) to represent netplan yaml. @@ -51,7 +52,8 @@ 'mtu': 1480, 'netmask': 64, 'type': 'static'}], - 'type: physical' + 'type: physical', + 'accept-ra': 'true' } An entry dictionary looks like: @@ -66,7 +68,7 @@ 'match': {'macaddress': '52:54:00:12:34:00'}, 'mtu': 1501, 'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"], - 'mtu6': 1480} + 'ipv6-mtu': 1480} """ @@ -79,6 +81,8 @@ else: return [obj, ] + if features is None: + features = [] addresses = [] routes = [] nameservers = [] @@ -92,6 +96,8 @@ if sn_type == 'dhcp': sn_type += '4' entry.update({sn_type: True}) + elif sn_type in IPV6_DYNAMIC_TYPES: + entry.update({'dhcp6': True}) elif sn_type in ['static']: addr = "%s" % subnet.get('address') if 'prefix' in subnet: @@ -108,8 +114,8 @@ searchdomains += _listify(subnet.get('dns_search', [])) if 'mtu' in subnet: mtukey = 'mtu' - if subnet_is_ipv6(subnet): - mtukey += '6' + if subnet_is_ipv6(subnet) and 'ipv6-mtu' in features: + mtukey = 'ipv6-mtu' entry.update({mtukey: subnet.get('mtu')}) for route in subnet.get('routes', []): to_net = "%s/%s" % (route.get('network'), @@ -144,6 +150,8 @@ ns = entry.get('nameservers', {}) ns.update({'search': searchdomains}) entry.update({'nameservers': ns}) + if 'accept-ra' in config and config['accept-ra'] is not None: + entry.update({'accept-ra': util.is_true(config.get('accept-ra'))}) def _extract_bond_slaves_by_name(interfaces, entry, bond_master): @@ -179,6 +187,7 @@ """Renders network information in a /etc/netplan/network.yaml format.""" NETPLAN_GENERATE = ['netplan', 'generate'] + NETPLAN_INFO = ['netplan', 'info'] def __init__(self, config=None): if not config: @@ -188,6 +197,22 @@ self.netplan_header = config.get('netplan_header', None) self._postcmds = config.get('postcmds', False) self.clean_default = config.get('clean_default', True) + self._features = config.get('features', None) + + @property + def features(self): + if self._features is None: + try: + info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True) + info = util.load_yaml(info_blob) + self._features = info['netplan.io']['features'] + except util.ProcessExecutionError: + # if the info subcommand is not present then we don't have any + # new features + pass + except (TypeError, KeyError) as e: + LOG.debug('Failed to list features from netplan info: %s', e) + return self._features def render_network_state(self, network_state, templates=None, target=None): # check network state for version @@ -235,9 +260,9 @@ # if content already in netplan format, pass it back if network_state.version == 2: LOG.debug('V2 to V2 passthrough') - return util.yaml_dumps({'network': network_state.config}, - explicit_start=False, - explicit_end=False) + return safeyaml.dumps({'network': network_state.config}, + explicit_start=False, + explicit_end=False) ethernets = {} wifis = {} @@ -271,7 +296,7 @@ else: del eth['match'] del eth['set-name'] - _extract_addresses(ifcfg, eth, ifname) + _extract_addresses(ifcfg, eth, ifname, self.features) ethernets.update({ifname: eth}) elif if_type == 'bond': @@ -296,7 +321,7 @@ slave_interfaces = ifcfg.get('bond-slaves') if slave_interfaces == 'none': _extract_bond_slaves_by_name(interfaces, bond, ifname) - _extract_addresses(ifcfg, bond, ifname) + _extract_addresses(ifcfg, bond, ifname, self.features) bonds.update({ifname: bond}) elif if_type == 'bridge': @@ -331,7 +356,7 @@ bridge.update({'parameters': br_config}) if ifcfg.get('mac_address'): bridge['macaddress'] = ifcfg.get('mac_address').lower() - _extract_addresses(ifcfg, bridge, ifname) + _extract_addresses(ifcfg, bridge, ifname, self.features) bridges.update({ifname: bridge}) elif if_type == 'vlan': @@ -343,7 +368,7 @@ macaddr = ifcfg.get('mac_address', None) if macaddr is not None: vlan['macaddress'] = macaddr.lower() - _extract_addresses(ifcfg, vlan, ifname) + _extract_addresses(ifcfg, vlan, ifname, self.features) vlans.update({ifname: vlan}) # inject global nameserver values under each all interface which @@ -359,10 +384,10 @@ # workaround yaml dictionary key sorting when dumping def _render_section(name, section): if section: - dump = util.yaml_dumps({name: section}, - explicit_start=False, - explicit_end=False, - noalias=True) + dump = safeyaml.dumps({name: section}, + explicit_start=False, + explicit_end=False, + noalias=True) txt = util.indent(dump, ' ' * 4) return [txt] return [] diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/net/network_state.py cloud-init-19.3-41-gc4735dd3/cloudinit/net/network_state.py --- cloud-init-19.2-36-g059d049c/cloudinit/net/network_state.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/net/network_state.py 2019-12-03 21:45:14.000000000 +0000 @@ -12,17 +12,23 @@ import six +from cloudinit import safeyaml from cloudinit import util LOG = logging.getLogger(__name__) NETWORK_STATE_VERSION = 1 +IPV6_DYNAMIC_TYPES = ['dhcp6', + 'ipv6_slaac', + 'ipv6_dhcpv6-stateless', + 'ipv6_dhcpv6-stateful'] NETWORK_STATE_REQUIRED_KEYS = { 1: ['version', 'config', 'network_state'], } NETWORK_V2_KEY_FILTER = [ - 'addresses', 'dhcp4', 'dhcp6', 'gateway4', 'gateway6', 'interfaces', - 'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan' + 'addresses', 'dhcp4', 'dhcp4-overrides', 'dhcp6', 'dhcp6-overrides', + 'gateway4', 'gateway6', 'interfaces', 'match', 'mtu', 'nameservers', + 'renderer', 'set-name', 'wakeonlan', 'accept-ra' ] NET_CONFIG_TO_V2 = { @@ -253,7 +259,7 @@ 'config': self._config, 'network_state': self._network_state, } - return util.yaml_dumps(state) + return safeyaml.dumps(state) def load(self, state): if 'version' not in state: @@ -272,7 +278,7 @@ setattr(self, key, state[key]) def dump_network_state(self): - return util.yaml_dumps(self._network_state) + return safeyaml.dumps(self._network_state) def as_dict(self): return {'version': self._version, 'config': self._config} @@ -340,7 +346,8 @@ 'name': 'eth0', 'subnets': [ {'type': 'dhcp4'} - ] + ], + 'accept-ra': 'true' } ''' @@ -360,6 +367,9 @@ self.use_ipv6 = True break + accept_ra = command.get('accept-ra', None) + if accept_ra is not None: + accept_ra = util.is_true(accept_ra) iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -370,6 +380,7 @@ 'address': None, 'gateway': None, 'subnets': subnets, + 'accept-ra': accept_ra }) self._network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @@ -613,6 +624,7 @@ driver: ixgbe set-name: lom1 dhcp6: true + accept-ra: true switchports: match: name: enp2* @@ -641,7 +653,7 @@ driver = match.get('driver', None) if driver: phy_cmd['params'] = {'driver': driver} - for key in ['mtu', 'match', 'wakeonlan']: + for key in ['mtu', 'match', 'wakeonlan', 'accept-ra']: if key in cfg: phy_cmd[key] = cfg[key] @@ -746,12 +758,20 @@ def _v2_to_v1_ipcfg(self, cfg): """Common ipconfig extraction from v2 to v1 subnets array.""" + def _add_dhcp_overrides(overrides, subnet): + if 'route-metric' in overrides: + subnet['metric'] = overrides['route-metric'] + subnets = [] if cfg.get('dhcp4'): - subnets.append({'type': 'dhcp4'}) + subnet = {'type': 'dhcp4'} + _add_dhcp_overrides(cfg.get('dhcp4-overrides', {}), subnet) + subnets.append(subnet) if cfg.get('dhcp6'): + subnet = {'type': 'dhcp6'} self.use_ipv6 = True - subnets.append({'type': 'dhcp6'}) + _add_dhcp_overrides(cfg.get('dhcp6-overrides', {}), subnet) + subnets.append(subnet) gateway4 = None gateway6 = None @@ -918,8 +938,9 @@ def subnet_is_ipv6(subnet): """Common helper for checking network_state subnets for ipv6.""" - # 'static6' or 'dhcp6' - if subnet['type'].endswith('6'): + # 'static6', 'dhcp6', 'ipv6_dhcpv6-stateful', 'ipv6_dhcpv6-stateless' or + # 'ipv6_slaac' + if subnet['type'].endswith('6') or subnet['type'] in IPV6_DYNAMIC_TYPES: # This is a request for DHCPv6. return True elif subnet['type'] == 'static' and is_ipv6_addr(subnet.get('address')): diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/net/sysconfig.py cloud-init-19.3-41-gc4735dd3/cloudinit/net/sysconfig.py --- cloud-init-19.2-36-g059d049c/cloudinit/net/sysconfig.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/net/sysconfig.py 2019-12-03 21:45:14.000000000 +0000 @@ -14,12 +14,11 @@ from . import renderer from .network_state import ( - is_ipv6_addr, net_prefix_to_ipv4_mask, subnet_is_ipv6) + is_ipv6_addr, net_prefix_to_ipv4_mask, subnet_is_ipv6, IPV6_DYNAMIC_TYPES) LOG = logging.getLogger(__name__) NM_CFG_FILE = "/etc/NetworkManager/NetworkManager.conf" -KNOWN_DISTROS = [ - 'opensuse', 'sles', 'suse', 'redhat', 'fedora', 'centos'] +KNOWN_DISTROS = ['centos', 'fedora', 'rhel', 'suse'] def _make_header(sep='#'): @@ -331,10 +330,14 @@ old_value = iface.get(old_key) if old_value is not None: # only set HWADDR on physical interfaces - if old_key == 'mac_address' and iface['type'] != 'physical': + if (old_key == 'mac_address' and + iface['type'] not in ['physical', 'infiniband']): continue iface_cfg[new_key] = old_value + if iface['accept-ra'] is not None: + iface_cfg['IPV6_FORCE_ACCEPT_RA'] = iface['accept-ra'] + @classmethod def _render_subnets(cls, iface_cfg, subnets, has_default_route): # setting base values @@ -344,10 +347,24 @@ for i, subnet in enumerate(subnets, start=len(iface_cfg.children)): mtu_key = 'MTU' subnet_type = subnet.get('type') - if subnet_type == 'dhcp6': + if subnet_type == 'dhcp6' or subnet_type == 'ipv6_dhcpv6-stateful': # TODO need to set BOOTPROTO to dhcp6 on SUSE iface_cfg['IPV6INIT'] = True + # Configure network settings using DHCPv6 + iface_cfg['DHCPV6C'] = True + elif subnet_type == 'ipv6_dhcpv6-stateless': + iface_cfg['IPV6INIT'] = True + # Configure network settings using SLAAC from RAs and optional + # info from dhcp server using DHCPv6 + iface_cfg['IPV6_AUTOCONF'] = True iface_cfg['DHCPV6C'] = True + # Use Information-request to get only stateless configuration + # parameters (i.e., without address). + iface_cfg['DHCPV6C_OPTIONS'] = '-S' + elif subnet_type == 'ipv6_slaac': + iface_cfg['IPV6INIT'] = True + # Configure network settings using SLAAC from RAs + iface_cfg['IPV6_AUTOCONF'] = True elif subnet_type in ['dhcp4', 'dhcp']: iface_cfg['BOOTPROTO'] = 'dhcp' elif subnet_type == 'static': @@ -390,10 +407,18 @@ ipv6_index = -1 for i, subnet in enumerate(subnets, start=len(iface_cfg.children)): subnet_type = subnet.get('type') + # metric may apply to both dhcp and static config + if 'metric' in subnet: + iface_cfg['METRIC'] = subnet['metric'] + # TODO(hjensas): Including dhcp6 here is likely incorrect. DHCPv6 + # does not ever provide a default gateway, the default gateway + # come from RA's. (https://github.com/openSUSE/wicked/issues/570) if subnet_type in ['dhcp', 'dhcp4', 'dhcp6']: if has_default_route and iface_cfg['BOOTPROTO'] != 'none': iface_cfg['DHCLIENT_SET_DEFAULT_ROUTE'] = False continue + elif subnet_type in IPV6_DYNAMIC_TYPES: + continue elif subnet_type == 'static': if subnet_is_ipv6(subnet): ipv6_index = ipv6_index + 1 @@ -421,9 +446,6 @@ else: iface_cfg['GATEWAY'] = subnet['gateway'] - if 'metric' in subnet: - iface_cfg['METRIC'] = subnet['metric'] - if 'dns_search' in subnet: iface_cfg['DOMAIN'] = ' '.join(subnet['dns_search']) @@ -439,10 +461,14 @@ @classmethod def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets): for _, subnet in enumerate(subnets, start=len(iface_cfg.children)): + subnet_type = subnet.get('type') for route in subnet.get('routes', []): is_ipv6 = subnet.get('ipv6') or is_ipv6_addr(route['gateway']) - if _is_default_route(route): + # Any dynamic configuration method, slaac, dhcpv6-stateful/ + # stateless should get router information from router RA's. + if (_is_default_route(route) and subnet_type not in + IPV6_DYNAMIC_TYPES): if ( (subnet.get('ipv4') and route_cfg.has_set_default_ipv4) or @@ -461,10 +487,17 @@ # TODO(harlowja): add validation that no other iface has # also provided the default route? iface_cfg['DEFROUTE'] = True + # TODO(hjensas): Including dhcp6 here is likely incorrect. + # DHCPv6 does not ever provide a default gateway, the + # default gateway come from RA's. + # (https://github.com/openSUSE/wicked/issues/570) if iface_cfg['BOOTPROTO'] in ('dhcp', 'dhcp4', 'dhcp6'): + # NOTE(hjensas): DHCLIENT_SET_DEFAULT_ROUTE is SuSE + # only. RHEL, CentOS, Fedora does not implement this + # option. iface_cfg['DHCLIENT_SET_DEFAULT_ROUTE'] = True if 'gateway' in route: - if is_ipv6 or is_ipv6_addr(route['gateway']): + if is_ipv6: iface_cfg['IPV6_DEFAULTGW'] = route['gateway'] route_cfg.has_set_default_ipv6 = True else: @@ -578,6 +611,10 @@ @staticmethod def _render_dns(network_state, existing_dns_path=None): + # skip writing resolv.conf if network_state doesn't include any input. + if not any([len(network_state.dns_nameservers), + len(network_state.dns_searchdomains)]): + return None content = resolv_conf.ResolvConf("") if existing_dns_path and os.path.isfile(existing_dns_path): content = resolv_conf.ResolvConf(util.load_file(existing_dns_path)) @@ -585,8 +622,6 @@ content.add_nameserver(nameserver) for searchdomain in network_state.dns_searchdomains: content.add_search_domain(searchdomain) - if not str(content): - return None header = _make_header(';') content_str = str(content) if not content_str.startswith(header): @@ -731,7 +766,7 @@ def available(target=None): sysconfig = available_sysconfig(target=target) nm = available_nm(target=target) - return (util.get_linux_distro()[0] in KNOWN_DISTROS + return (util.system_info()['variant'] in KNOWN_DISTROS and any([nm, sysconfig])) @@ -744,11 +779,11 @@ expected_paths = [ 'etc/sysconfig/network-scripts/network-functions', - 'etc/sysconfig/network-scripts/ifdown-eth'] + 'etc/sysconfig/config'] for p in expected_paths: - if not os.path.isfile(util.target_path(target, p)): - return False - return True + if os.path.isfile(util.target_path(target, p)): + return True + return False def available_nm(target=None): diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/net/tests/test_init.py cloud-init-19.3-41-gc4735dd3/cloudinit/net/tests/test_init.py --- cloud-init-19.2-36-g059d049c/cloudinit/net/tests/test_init.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/net/tests/test_init.py 2019-12-03 21:45:14.000000000 +0000 @@ -157,11 +157,40 @@ ensure_file(os.path.join(self.sysdir, 'eth0', 'bonding')) self.assertTrue(net.is_bond('eth0')) - def test_has_master(self): - """has_master is True when /sys/net/devname/master exists.""" - self.assertFalse(net.has_master('enP1s1')) - ensure_file(os.path.join(self.sysdir, 'enP1s1', 'master')) - self.assertTrue(net.has_master('enP1s1')) + def test_get_master(self): + """get_master returns the path when /sys/net/devname/master exists.""" + self.assertIsNone(net.get_master('enP1s1')) + master_path = os.path.join(self.sysdir, 'enP1s1', 'master') + ensure_file(master_path) + self.assertEqual(master_path, net.get_master('enP1s1')) + + def test_master_is_bridge_or_bond(self): + bridge_mac = 'aa:bb:cc:aa:bb:cc' + bond_mac = 'cc:bb:aa:cc:bb:aa' + + # No master => False + write_file(os.path.join(self.sysdir, 'eth1', 'address'), bridge_mac) + write_file(os.path.join(self.sysdir, 'eth2', 'address'), bond_mac) + + self.assertFalse(net.master_is_bridge_or_bond('eth1')) + self.assertFalse(net.master_is_bridge_or_bond('eth2')) + + # masters without bridge/bonding => False + write_file(os.path.join(self.sysdir, 'br0', 'address'), bridge_mac) + write_file(os.path.join(self.sysdir, 'bond0', 'address'), bond_mac) + + os.symlink('../br0', os.path.join(self.sysdir, 'eth1', 'master')) + os.symlink('../bond0', os.path.join(self.sysdir, 'eth2', 'master')) + + self.assertFalse(net.master_is_bridge_or_bond('eth1')) + self.assertFalse(net.master_is_bridge_or_bond('eth2')) + + # masters with bridge/bonding => True + write_file(os.path.join(self.sysdir, 'br0', 'bridge'), '') + write_file(os.path.join(self.sysdir, 'bond0', 'bonding'), '') + + self.assertTrue(net.master_is_bridge_or_bond('eth1')) + self.assertTrue(net.master_is_bridge_or_bond('eth2')) def test_is_vlan(self): """is_vlan is True when /sys/net/devname/uevent has DEVTYPE=vlan.""" @@ -461,6 +490,26 @@ expected = [('ens3', mac, None, None)] self.assertEqual(expected, net.get_interfaces()) + def test_get_interfaces_does_not_skip_phys_members_of_bridges_and_bonds( + self + ): + bridge_mac = 'aa:bb:cc:aa:bb:cc' + bond_mac = 'cc:bb:aa:cc:bb:aa' + write_file(os.path.join(self.sysdir, 'br0', 'address'), bridge_mac) + write_file(os.path.join(self.sysdir, 'br0', 'bridge'), '') + + write_file(os.path.join(self.sysdir, 'bond0', 'address'), bond_mac) + write_file(os.path.join(self.sysdir, 'bond0', 'bonding'), '') + + write_file(os.path.join(self.sysdir, 'eth1', 'address'), bridge_mac) + os.symlink('../br0', os.path.join(self.sysdir, 'eth1', 'master')) + + write_file(os.path.join(self.sysdir, 'eth2', 'address'), bond_mac) + os.symlink('../bond0', os.path.join(self.sysdir, 'eth2', 'master')) + + interface_names = [interface[0] for interface in net.get_interfaces()] + self.assertEqual(['eth1', 'eth2'], sorted(interface_names)) + class TestInterfaceHasOwnMAC(CiTestCase): diff -Nru cloud-init-19.2-36-g059d049c/cloudinit/reporting/handlers.py cloud-init-19.3-41-gc4735dd3/cloudinit/reporting/handlers.py --- cloud-init-19.2-36-g059d049c/cloudinit/reporting/handlers.py 2019-09-17 10:11:00.000000000 +0000 +++ cloud-init-19.3-41-gc4735dd3/cloudinit/reporting/handlers.py 2019-12-03 21:45:14.000000000 +0000 @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import abc +import uuid import fcntl import json import six @@ -201,10 +202,11 @@ def _event_key(self, event): """ the event key format is: - CLOUD_INIT||| + CLOUD_INIT||||