diff -Nru cloud-init-20.1-10-g71af48df/ChangeLog cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/ChangeLog --- cloud-init-20.1-10-g71af48df/ChangeLog 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/ChangeLog 2021-05-11 16:34:29.000000000 +0000 @@ -1,3 +1,608 @@ +21.2 + - Add \r\n check for SSH keys in Azure (#889) + - Revert "Add support to resize rootfs if using LVM (#721)" (#887) + (LP: #1922742) + - Add Vultaire as contributor (#881) [Paul Goins] + - Azure: adding support for consuming userdata from IMDS (#884) [Anh Vo] + - test_upgrade: modify test_upgrade_package to run for more sources (#883) + - Fix chef module run failure when chef_license is set (#868) [Ben Hughes] + - Azure: Retry net metadata during nic attach for non-timeout errs (#878) + [aswinrajamannar] + - Azure: Retrieve username and hostname from IMDS (#865) [Thomas Stringer] + - Azure: eject the provisioning iso before reporting ready (#861) [Anh Vo] + - Use `partprobe` to re-read partition table if available (#856) + [Nicolas Bock] (LP: #1920939) + - fix error on upgrade caused by new vendordata2 attributes (#869) + (LP: #1922739) + - add prefer_fqdn_over_hostname config option (#859) + [hamalq] (LP: #1921004) + - Emit dots on travis to avoid timeout (#867) + - doc: Replace remaining references to user-scripts as a config module + (#866) [Ryan Harper] + - azure: Removing ability to invoke walinuxagent (#799) [Anh Vo] + - Add Vultr support (#827) [David Dymko] + - Fix unpickle for source paths missing run_dir (#863) + [lucasmoura] (LP: #1899299) + - sysconfig: use BONDING_MODULE_OPTS on SUSE (#831) [Jens Sandmann] + - bringup_static_routes: fix gateway check (#850) [Petr Fedchenkov] + - add hamalq user (#860) [hamalq] + - Add support to resize rootfs if using LVM (#721) + [Eduardo Otubo] (LP: #1799953) + - Fix mis-detecting network configuration in initramfs cmdline (#844) + (LP: #1919188) + - tools/write-ssh-key-fingerprints: do not display empty header/footer + (#817) [dermotbradley] + - Azure helper: Ensure Azure http handler sleeps between retries (#842) + [Johnson Shi] + - Fix chef apt source example (#826) [timothegenzmer] + - .travis.yml: generate an SSH key before running tests (#848) + - write passwords only to serial console, lock down cloud-init-output.log + (#847) (LP: #1918303) + - Fix apt default integration test (#845) + - integration_tests: bump pycloudlib dependency (#846) + - Fix stack trace if vendordata_raw contained an array (#837) [eb3095] + - archlinux: Fix broken locale logic (#841) + [Kristian Klausen] (LP: #1402406) + - Integration test for #783 (#832) + - integration_tests: mount more paths IN_PLACE (#838) + - Fix requiring device-number on EC2 derivatives (#836) (LP: #1917875) + - Remove the vi comment from the part-handler example (#835) + - net: exclude OVS internal interfaces in get_interfaces (#829) + (LP: #1912844) + - tox.ini: pass OS_* environment variables to integration tests (#830) + - integration_tests: add OpenStack as a platform (#804) + - Add flexibility to IMDS api-version (#793) [Thomas Stringer] + - Fix the TestApt tests using apt-key on Xenial and Hirsute (#823) + [Paride Legovini] (LP: #1916629) + - doc: remove duplicate "it" from nocloud.rst (#825) [V.I. Wood] + - archlinux: Use hostnamectl to set the transient hostname (#797) + [Kristian Klausen] + - cc_keys_to_console.py: Add documentation for recently added config key + (#824) [dermotbradley] + - Update cc_set_hostname documentation (#818) [Toshi Aoyama] + +21.1 + - Azure: Support for VMs without ephemeral resource disks. (#800) + [Johnson Shi] (LP: #1901011) + - cc_keys_to_console: add option to disable key emission (#811) + [Michael Hudson-Doyle] (LP: #1915460) + - integration_tests: introduce lxd_use_exec mark (#802) + - azure: case-insensitive UUID to avoid new IID during kernel upgrade + (#798) (LP: #1835584) + - stale.yml: don't ask submitters to reopen PRs (#816) + - integration_tests: fix use of SSH agent within tox (#815) + - integration_tests: add UPGRADE CloudInitSource (#812) + - integration_tests: use unique MAC addresses for tests (#813) + - Update .gitignore (#814) + - Port apt cloud_tests to integration tests (#808) + - integration_tests: fix test_gh626 on LXD VMs (#809) + - Fix attempting to decode binary data in test_seed_random_data test (#806) + - Remove wait argument from tests with session_cloud calls (#805) + - Datasource for UpCloud (#743) [Antti Myyrä] + - test_gh668: fix failure on LXD VMs (#801) + - openstack: read the dynamic metadata group vendor_data2.json (#777) + [Andrew Bogott] (LP: #1841104) + - includedir in suoders can be prefixed by "arroba" (#783) + [Jordi Massaguer Pla] + - [VMware] change default max wait time to 15s (#774) [xiaofengw-vmware] + - Revert integration test associated with reverted #586 (#784) + - Add jordimassaguerpla as contributor (#787) [Jordi Massaguer Pla] + - Add Rick Harding to CLA signers (#792) [Rick Harding] + - HACKING.rst: add clarifying note to LP CLA process section (#789) + - Stop linting cloud_tests (#791) + - cloud-tests: update cryptography requirement (#790) [Joshua Powers] + - Remove 'remove-raise-on-failure' calls from integration_tests (#788) + - Use more cloud defaults in integration tests (#757) + - Adding self to cla signers (#776) [Andrew Bogott] + - doc: avoid two warnings (#781) [Dan Kenigsberg] + - Use proper spelling for Red Hat (#778) [Dan Kenigsberg] + - Add antonyc to .github-cla-signers (#747) [Anton Chaporgin] + - integration_tests: log image serial if available (#772) + - [VMware] Support cloudinit raw data feature (#691) [xiaofengw-vmware] + - net: Fix static routes to host in eni renderer (#668) [Pavel Abalikhin] + - .travis.yml: don't run cloud_tests in CI (#756) + - test_upgrade: add some missing commas (#769) + - cc_seed_random: update documentation and fix integration test (#771) + (LP: #1911227) + - Fix test gh-632 test to only run on NoCloud (#770) (LP: #1911230) + - archlinux: fix package upgrade command handling (#768) [Bao Trinh] + - integration_tests: add integration test for LP: #1910835 (#761) + - Fix regression with handling of IMDS ssh keys (#760) [Thomas Stringer] + - integration_tests: log cloud-init version in SUT (#758) + - Add ajmyyra as contributor (#742) [Antti Myyrä] + - net_convert: add some missing help text (#755) + - Missing IPV6_AUTOCONF=no to render sysconfig dhcp6 stateful on RHEL + (#753) [Eduardo Otubo] + - doc: document missing IPv6 subnet types (#744) [Antti Myyrä] + - Add example configuration for datasource `AliYun` (#751) [Xiaoyu Zhong] + - integration_tests: add SSH key selection settings (#754) + - fix a typo in man page cloud-init.1 (#752) [Amy Chen] + - network-config-format-v2.rst: add Netplan Passthrough section (#750) + - stale: re-enable post holidays (#749) + - integration_tests: port ca_certs tests from cloud_tests (#732) + - Azure: Add telemetry for poll IMDS (#741) [Johnson Shi] + - doc: move testing section from HACKING to its own doc (#739) + - No longer allow integration test failures on travis (#738) + - stale: fix error in definition (#740) + - integration_tests: set log-cli-level to INFO by default (#737) + - PULL_REQUEST_TEMPLATE.md: use backticks around commit message (#736) + - stale: disable check for holiday break (#735) + - integration_tests: log the path we collect logs into (#733) + - .travis.yml: add (most) supported Python versions to CI (#734) + - integration_tests: fix IN_PLACE CLOUD_INIT_SOURCE (#731) + - cc_ca_certs: add RHEL support (#633) [cawamata] + - Azure: only generate config for NICs with addresses (#709) + [Thomas Stringer] + - doc: fix CloudStack configuration example (#707) [Olivier Lemasle] + - integration_tests: restrict test_lxd_bridge appropriately (#730) + - Add integration tests for CLI functionality (#729) + - Integration test for gh-626 (#728) + - Some test_upgrade fixes (#726) + - Ensure overriding test vars with env vars works for booleans (#727) + - integration_tests: port lxd_bridge test from cloud_tests (#718) + - Integration test for gh-632. (#725) + - Integration test for gh-671 (#724) + - integration-requirements.txt: bump pycloudlib commit (#723) + - Drop unnecessary shebang from cmd/main.py (#722) [Eduardo Otubo] + - Integration test for LP: #1813396 and #669 (#719) + - integration_tests: include timestamp in log output (#720) + - integration_tests: add test for LP: #1898997 (#713) + - Add integration test for power_state_change module (#717) + - Update documentation for network-config-format-v2 (#701) [ggiesen] + - sandbox CA Cert tests to not require ca-certificates (#715) + [Eduardo Otubo] + - Add upgrade integration test (#693) + - Integration test for 570 (#712) + - Add ability to keep snapshotted images in integration tests (#711) + - Integration test for pull #586 (#706) + - integration_tests: introduce skipping of tests by OS (#702) + - integration_tests: introduce IntegrationInstance.restart (#708) + - Add lxd-vm to list of valid integration test platforms (#705) + - Adding BOOTPROTO = dhcp to render sysconfig dhcp6 stateful on RHEL + (#685) [Eduardo Otubo] + - Delete image snapshots created for integration tests (#682) + - Parametrize ssh_keys_provided integration test (#700) [lucasmoura] + - Drop use_sudo attribute on IntegrationInstance (#694) [lucasmoura] + - cc_apt_configure: add riscv64 as a ports arch (#687) + [Dimitri John Ledkov] + - cla: add xnox (#692) [Dimitri John Ledkov] + - Collect logs from integration test runs (#675) + +20.4.1 + - Revert "ssh_util: handle non-default AuthorizedKeysFile config (#586)" + +20.4 + - tox: avoid tox testenv subsvars for xenial support (#684) + - Ensure proper root permissions in integration tests (#664) [James Falcon] + - LXD VM support in integration tests (#678) [James Falcon] + - Integration test for fallocate falling back to dd (#681) [James Falcon] + - .travis.yml: correctly integration test the built .deb (#683) + - Ability to hot-attach NICs to preprovisioned VMs before reprovisioning + (#613) [aswinrajamannar] + - Support configuring SSH host certificates. (#660) [Jonathan Lung] + - add integration test for LP: #1900837 (#679) + - cc_resizefs on FreeBSD: Fix _can_skip_ufs_resize (#655) + [Mina Galić] (LP: #1901958, #1901958) + - DataSourceAzure: push dmesg log to KVP (#670) [Anh Vo] + - Make mount in place for tests work (#667) [James Falcon] + - integration_tests: restore emission of settings to log (#657) + - DataSourceAzure: update password for defuser if exists (#671) [Anh Vo] + - tox.ini: only select "ci" marked tests for CI runs (#677) + - Azure helper: Increase Azure Endpoint HTTP retries (#619) [Johnson Shi] + - DataSourceAzure: send failure signal on Azure datasource failure (#594) + [Johnson Shi] + - test_persistence: simplify VersionIsPoppedFromState (#674) + - only run a subset of integration tests in CI (#672) + - cli: add --system param to allow validating system user-data on a + machine (#575) + - test_persistence: add VersionIsPoppedFromState test (#673) + - introduce an upgrade framework and related testing (#659) + - add --no-tty option to gpg (#669) [Till Riedel] (LP: #1813396) + - Pin pycloudlib to a working commit (#666) [James Falcon] + - DataSourceOpenNebula: exclude SRANDOM from context output (#665) + - cloud_tests: add hirsute release definition (#662) + - split integration and cloud_tests requirements (#652) + - faq.rst: add warning to answer that suggests running `clean` (#661) + - Fix stacktrace in DataSourceRbxCloud if no metadata disk is found (#632) + [Scott Moser] + - Make wakeonlan Network Config v2 setting actually work (#626) + [dermotbradley] + - HACKING.md: unify network-refactoring namespace (#658) [Mina Galić] + - replace usage of dmidecode with kenv on FreeBSD (#621) [Mina Galić] + - Prevent timeout on travis integration tests. (#651) [James Falcon] + - azure: enable pushing the log to KVP from the last pushed byte (#614) + [Moustafa Moustafa] + - Fix launch_kwargs bug in integration tests (#654) [James Falcon] + - split read_fs_info into linux & freebsd parts (#625) [Mina Galić] + - PULL_REQUEST_TEMPLATE.md: expand commit message section (#642) + - Make some language improvements in growpart documentation (#649) + [Shane Frasier] + - Revert ".travis.yml: use a known-working version of lxd (#643)" (#650) + - Fix not sourcing default 50-cloud-init ENI file on Debian (#598) + [WebSpider] + - remove unnecessary reboot from gpart resize (#646) [Mina Galić] + - cloudinit: move dmi functions out of util (#622) [Scott Moser] + - integration_tests: various launch improvements (#638) + - test_lp1886531: don't assume /etc/fstab exists (#639) + - Remove Ubuntu restriction from PR template (#648) [James Falcon] + - util: fix mounting of vfat on *BSD (#637) [Mina Galić] + - conftest: improve docstring for disable_subp_usage (#644) + - doc: add example query commands to debug Jinja templates (#645) + - Correct documentation and testcase data for some user-data YAML (#618) + [dermotbradley] + - Hetzner: Fix instance_id / SMBIOS serial comparison (#640) + [Markus Schade] + - .travis.yml: use a known-working version of lxd (#643) + - tools/build-on-freebsd: fix comment explaining purpose of the script + (#635) [Mina Galić] + - Hetzner: initialize instance_id from system-serial-number (#630) + [Markus Schade] (LP: #1885527) + - Explicit set IPV6_AUTOCONF and IPV6_FORCE_ACCEPT_RA on static6 (#634) + [Eduardo Otubo] + - get_interfaces: don't exclude Open vSwitch bridge/bond members (#608) + [Lukas Märdian] (LP: #1898997) + - Add config modules for controlling IBM PowerVM RMC. (#584) + [Aman306] (LP: #1895979) + - Update network config docs to clarify MAC address quoting (#623) + [dermotbradley] + - gentoo: fix hostname rendering when value has a comment (#611) + [Manuel Aguilera] + - refactor integration testing infrastructure (#610) [James Falcon] + - stages: don't reset permissions of cloud-init.log every boot (#624) + (LP: #1900837) + - docs: Add how to use cloud-localds to boot qemu (#617) [Joshua Powers] + - Drop vestigial update_resolve_conf_file function (#620) [Scott Moser] + - cc_mounts: correctly fallback to dd if fallocate fails (#585) + (LP: #1897099) + - .travis.yml: add integration-tests to Travis matrix (#600) + - ssh_util: handle non-default AuthorizedKeysFile config (#586) + [Eduardo Otubo] + - Multiple file fix for AuthorizedKeysFile config (#60) [Eduardo Otubo] + - bddeb: new --packaging-branch argument to pull packaging from branch + (#576) [Paride Legovini] + - Add more integration tests (#615) [lucasmoura] + - DataSourceAzure: write marker file after report ready in preprovisioning + (#590) [Johnson Shi] + - integration_tests: emit settings to log during setup (#601) + - integration_tests: implement citest tests run in Travis (#605) + - Add Azure support to integration test framework (#604) [James Falcon] + - openstack: consider product_name as valid chassis tag (#580) + [Adrian Vladu] (LP: #1895976) + - azure: clean up and refactor report_diagnostic_event (#563) [Johnson Shi] + - net: add the ability to blacklist network interfaces based on driver + during enumeration of physical network devices (#591) [Anh Vo] + - integration_tests: don't error on cloud-init failure (#596) + - integration_tests: improve cloud-init.log assertions (#593) + - conftest.py: remove top-level import of httpretty (#599) + - tox.ini: add integration-tests testenv definition (#595) + - PULL_REQUEST_TEMPLATE.md: empty checkboxes need a space (#597) + - add integration test for LP: #1886531 (#592) + - Initial implementation of integration testing infrastructure (#581) + [James Falcon] + - Fix name of ntp and chrony service on CentOS and RHEL. (#589) + [Scott Moser] (LP: #1897915) + - Adding a PR template (#587) [James Falcon] + - Azure parse_network_config uses fallback cfg when generate IMDS network + cfg fails (#549) [Johnson Shi] + - features: refresh docs for easier out-of-context reading (#582) + - Fix typo in resolv_conf module's description (#578) [Wacław Schiller] + - cc_users_groups: minor doc formatting fix (#577) + - Fix typo in disk_setup module's description (#579) [Wacław Schiller] + - Add vendor-data support to seedfrom parameter for NoCloud and OVF (#570) + [Johann Queuniet] + - boot.rst: add First Boot Determination section (#568) (LP: #1888858) + - opennebula.rst: minor readability improvements (#573) [Mina Galić] + - cloudinit: remove unused LOG variables (#574) + - create a shutdown_command method in distro classes (#567) + [Emmanuel Thomé] + - user_data: remove unused constant (#566) + - network: Fix type and respect name when rendering vlan in + sysconfig. (#541) [Eduardo Otubo] (LP: #1788915, #1826608) + - Retrieve SSH keys from IMDS first with OVF as a fallback (#509) + [Thomas Stringer] + - Add jqueuniet as contributor (#569) [Johann Queuniet] + - distros: minor typo fix (#562) + - Bump the integration-requirements versioned dependencies (#565) + [Paride Legovini] + - network-config-format-v1: fix typo in nameserver example (#564) + [Stanislas] + - Run cloud-init-local.service after the hv_kvp_daemon (#505) + [Robert Schweikert] + - Add method type hints for Azure helper (#540) [Johnson Shi] + - systemd: add Before=shutdown.target when Conflicts=shutdown.target is + used (#546) [Paride Legovini] + - LXD: detach network from profile before deleting it (#542) + [Paride Legovini] (LP: #1776958) + - redhat spec: add missing BuildRequires (#552) [Paride Legovini] + - util: remove debug statement (#556) [Joshua Powers] + - Fix cloud config on chef example (#551) [lucasmoura] + +20.3 + - Azure: Add netplan driver filter when using hv_netvsc driver (#539) + [James Falcon] (LP: #1830740) + - query: do not handle non-decodable non-gzipped content (#543) + - DHCP sandboxing failing on noexec mounted /var/tmp (#521) [Eduardo Otubo] + - Update the list of valid ssh keys. (#487) + [Ole-Martin Bratteng] (LP: #1877869) + - cmd: cloud-init query to handle compressed userdata (#516) (LP: #1889938) + - Pushing cloud-init log to the KVP (#529) [Moustafa Moustafa] + - Add Alpine Linux support. (#535) [dermotbradley] + - Detect kernel version before swap file creation (#428) [Eduardo Otubo] + - cli: add devel make-mime subcommand (#518) + - user-data: only verify mime-types for TYPE_NEEDED and x-shellscript + (#511) (LP: #1888822) + - DataSourceOracle: retry twice (and document why we retry at all) (#536) + - Refactor Azure report ready code (#468) [Johnson Shi] + - tox.ini: pin correct version of httpretty in xenial{,-dev} envs (#531) + - Support Oracle IMDSv2 API (#528) [James Falcon] + - .travis.yml: run a doc build during CI (#534) + - doc/rtd/topics/datasources/ovf.rst: fix doc8 errors (#533) + - Fix 'Users and Groups' configuration documentation (#530) [sshedi] + - cloudinit.distros: update docstrings of add_user and create_user (#527) + - Fix headers for device types in network v2 docs (#532) + [Caleb Xavier Berger] + - Add AlexBaranowski as contributor (#508) [Aleksander Baranowski] + - DataSourceOracle: refactor to use only OPC v1 endpoint (#493) + - .github/workflows/stale.yml: s/Josh/Rick/ (#526) + - Fix a typo in apt pipelining module (#525) [Xiao Liang] + - test_util: parametrize devlist tests (#523) [James Falcon] + - Recognize LABEL_FATBOOT labels (#513) [James Falcon] (LP: #1841466) + - Handle additional identifier for SLES For HPC (#520) [Robert Schweikert] + - Revert "test-requirements.txt: pin pytest to <6 (#512)" (#515) + - test-requirements.txt: pin pytest to <6 (#512) + - Add "tsanghan" as contributor (#504) [tsanghan] + - fix brpm building (LP: #1886107) + - Adding eandersson as a contributor (#502) [Erik Olof Gunnar Andersson] + - azure: disable bouncing hostname when setting hostname fails (#494) + [Anh Vo] + - VMware: Support parsing DEFAULT-RUN-POST-CUST-SCRIPT (#441) + [xiaofengw-vmware] + - DataSourceAzure: Use ValueError when JSONDecodeError is not available + (#490) [Anh Vo] + - cc_ca_certs.py: fix blank line problem when removing CAs and adding + new one (#483) [dermotbradley] + - freebsd: py37-serial is now py37-pyserial (#492) [Gonéri Le Bouder] + - ssh exit with non-zero status on disabled user (#472) + [Eduardo Otubo] (LP: #1170059) + - cloudinit: remove global disable of pylint W0107 and fix errors (#489) + - networking: refactor wait_for_physdevs from cloudinit.net (#466) + (LP: #1884626) + - HACKING.rst: add pytest.param pytest gotcha (#481) + - cloudinit: remove global disable of pylint W0105 and fix errors (#480) + - Fix two minor warnings (#475) + - test_data: fix faulty patch (#476) + - cc_mounts: handle missing fstab (#484) (LP: #1886531) + - LXD cloud_tests: support more lxd image formats (#482) [Paride Legovini] + - Add update_etc_hosts as default module on *BSD (#479) [Adam Dobrawy] + - cloudinit: fix tip-pylint failures and bump pinned pylint version (#478) + - Added BirknerAlex as contributor and sorted the file (#477) + [Alexander Birkner] + - Update list of types of modules in cli.rst [saurabhvartak1982] + - tests: use markers to configure disable_subp_usage (#473) + - Add mention of vendor-data to no-cloud format documentation (#470) + [Landon Kirk] + - Fix broken link to OpenStack metadata service docs (#467) + [Matt Riedemann] + - Disable ec2 mirror for non aws instances (#390) + [lucasmoura] (LP: #1456277) + - cloud_tests: don't pass --python-version to read-dependencies (#465) + - networking: refactor is_physical from cloudinit.net (#457) (LP: #1884619) + - Enable use of the caplog fixture in pytest tests, and add a + cc_final_message test using it (#461) + - RbxCloud: Add support for FreeBSD (#464) [Adam Dobrawy] + - Add schema for cc_chef module (#375) [lucasmoura] (LP: #1858888) + - test_util: add (partial) testing for util.mount_cb (#463) + - .travis.yml: revert to installing ubuntu-dev-tools (#460) + - HACKING.rst: add details of net refactor tracking (#456) + - .travis.yml: rationalise installation of dependencies in host (#449) + - Add dermotbradley as contributor. (#458) [dermotbradley] + - net/networking: remove unused functions/methods (#453) + - distros.networking: initial implementation of layout (#391) + - cloud-init.service.tmpl: use "rhel" instead of "redhat" (#452) + - Change from redhat to rhel in systemd generator tmpl (#450) + [Eduardo Otubo] + - Hetzner: support reading user-data that is base64 encoded. (#448) + [Scott Moser] (LP: #1884071) + - HACKING.rst: add strpath gotcha to testing gotchas section (#446) + - cc_final_message: don't create directories when writing boot-finished + (#445) (LP: #1883903) + - .travis.yml: only store new schroot if something has changed (#440) + - util: add ensure_dir_exists parameter to write_file (#443) + - printing the error stream of the dhclient process before killing it + (#369) [Moustafa Moustafa] + - Fix link to the MAAS documentation (#442) + [Paride Legovini] (LP: #1883666) + - RPM build: disable the dynamic mirror URLs when using a proxy (#437) + [Paride Legovini] + - util: rename write_file's copy_mode parameter to preserve_mode (#439) + - .travis.yml: use $TRAVIS_BUILD_DIR for lxd_image caching (#438) + - cli.rst: alphabetise devel subcommands and add net-convert to list (#430) + - Default to UTF-8 in /var/log/cloud-init.log (#427) [James Falcon] + - travis: cache the chroot we use for package builds (#429) + - test: fix all flake8 E126 errors (#425) [Joshua Powers] + - Fixes KeyError for bridge with no "parameters:" setting (#423) + [Brian Candler] (LP: #1879673) + - When tools.conf does not exist, running cmd "vmware-toolbox-cmd + config get deployPkg enable-custom-scripts", the return code will + be EX_UNAVAILABLE(69), on this condition, it should not take it as + error. (#413) [chengcheng-chcheng] + - Document CloudStack data-server well-known hostname (#399) [Gregor Riepl] + - test: move conftest.py to top-level, to cover tests/ also (#414) + - Replace cc_chef is_installed with use of subp.is_exe. (#421) + [Scott Moser] + - Move runparts to subp. (#420) [Scott Moser] + - Move subp into its own module. (#416) [Scott Moser] + - readme: point at travis-ci.com (#417) [Joshua Powers] + - New feature flag functionality and fix includes failing silently (#367) + [James Falcon] (LP: #1734939) + - Enhance poll imds logging (#365) [Moustafa Moustafa] + - test: fix all flake8 E121 and E123 errors (#404) [Joshua Powers] + - test: fix all flake8 E241 (#403) [Joshua Powers] + - test: ignore flake8 E402 errors in main.py (#402) [Joshua Powers] + - cc_grub_dpkg: determine idevs in more robust manner with grub-probe + (#358) [Matthew Ruffell] (LP: #1877491) + - test: fix all flake8 E741 errors (#401) [Joshua Powers] + - tests: add groovy integration tests for ubuntu (#400) + - Enable chef_license support for chef infra client (#389) [Bipin Bachhao] + - testing: use flake8 again (#392) [Joshua Powers] + - enable Puppet, Chef mcollective in default config (#385) + [Mina Galić (deprecated: Igor Galić)] (LP: #1880279) + - HACKING.rst: introduce .net -> Networking refactor section (#384) + - Travis: do not install python3-contextlib2 (dropped dependency) (#388) + [Paride Legovini] + - HACKING: mention that .github-cla-signers is alpha-sorted (#380) + - Add bipinbachhao as contributor (#379) [Bipin Bachhao] + - cc_snap: validate that assertions property values are strings (#370) + - conftest: implement partial disable_subp_usage (#371) + - test_resolv_conf: refresh stale comment (#374) + - cc_snap: apply validation to snap.commands properties (#364) + - make finding libc platform independent (#366) + [Mina Galić (deprecated: Igor Galić)] + - doc/rtd/topics/faq: Updates LXD docs links to current site (#368) [TomP] + - templater: drop Jinja Python 2 compatibility shim (#353) + - cloudinit: minor pylint fixes (#360) + - cloudinit: remove unneeded __future__ imports (#362) + - migrating momousta lp user to Moustafa-Moustafa GitHub user (#361) + [Moustafa Moustafa] + - cloud_tests: emit dots on Travis while fetching images (#347) + - Add schema to apt configure config (#357) [lucasmoura] (LP: #1858884) + - conftest: add docs and tests regarding CiTestCase's subp functionality + (#343) + - analyze/dump: refactor shared string into variable (#350) + - doc: update boot.rst with correct timing of runcmd (#351) + - HACKING.rst: change contact info to Rick Harding (#359) [lucasmoura] + - HACKING.rst: guide people to add themselves to the CLA file (#349) + - HACKING.rst: more unit testing documentation (#354) + - .travis.yml: don't run lintian during integration test package builds + (#352) + - Add test to ensure docs examples are valid cloud-init configs (#355) + [James Falcon] (LP: #1876414) + - make suse and sles support 127.0.1.1 (#336) [chengcheng-chcheng] + - Create tests to validate schema examples (#348) + [lucasmoura] (LP: #1876412) + - analyze/dump: add support for Amazon Linux 2 log lines (#346) + (LP: #1876323) + - bsd: upgrade support (#305) [Gonéri Le Bouder] + - Add lucasmoura as contributor (#345) [lucasmoura] + - Add "therealfalcon" as contributor (#344) [James Falcon] + - Adapt the package building scripts to use Python 3 (#231) + [Paride Legovini] + - DataSourceEc2: use metadata's NIC ordering to determine route-metrics + (#342) (LP: #1876312) + - .travis.yml: introduce caching (#329) + - cc_locale: introduce schema (#335) + - doc/rtd/conf.py: bump copyright year to 2020 (#341) + - yum_add_repo: Add Centos to the supported distro list (#340) + +20.2 + - doc/format: reference make-mime.py instead of an inline script (#334) + - Add docs about creating parent folders (#330) [Adrian Wilkins] + - DataSourceNoCloud/OVF: drop claim to support FTP (#333) (LP: #1875470) + - schema: ignore spurious pylint error (#332) + - schema: add json schema for write_files module (#152) + - BSD: find_devs_with_ refactoring (#298) [Gonéri Le Bouder] + - nocloud: drop work around for Linux 2.6 (#324) [Gonéri Le Bouder] + - cloudinit: drop dependencies on unittest2 and contextlib2 (#322) + - distros: handle a potential mirror filtering error case (#328) + - log: remove unnecessary import fallback logic (#327) + - .travis.yml: don't run integration test on ubuntu/* branches (#321) + - More unit test documentation (#314) + - conftest: introduce disable_subp_usage autouse fixture (#304) + - YAML align indent sizes for docs readability (#323) [Tak Nishigori] + - network_state: add missing space to log message (#325) + - tests: add missing mocks for get_interfaces_by_mac (#326) (LP: #1873910) + - test_mounts: expand happy path test for both happy paths (#319) + - cc_mounts: fix incorrect format specifiers (#316) (LP: #1872836) + - swap file "size" being used before checked if str (#315) [Eduardo Otubo] + - HACKING.rst: add pytest version gotchas section (#311) + - docs: Add steps to re-run cloud-id and cloud-init (#313) [Joshua Powers] + - readme: OpenBSD is now supported (#309) [Gonéri Le Bouder] + - net: ignore 'renderer' key in netplan config (#306) (LP: #1870421) + - Add support for NFS/EFS mounts (#300) [Andrew Beresford] (LP: #1870370) + - openbsd: set_passwd should not unlock user (#289) [Gonéri Le Bouder] + - tools/.github-cla-signers: add beezly as CLA signer (#301) + - util: remove unnecessary lru_cache import fallback (#299) + - HACKING.rst: reorganise/update CLA signature info (#297) + - distros: drop leading/trailing hyphens from mirror URL labels (#296) + - HACKING.rst: add note about variable annotations (#295) + - CiTestCase: stop using and remove sys_exit helper (#283) + - distros: replace invalid characters in mirror URLs with hyphens (#291) + (LP: #1868232) + - rbxcloud: gracefully handle arping errors (#262) [Adam Dobrawy] + - Fix cloud-init ignoring some misdeclared mimetypes in user-data. + [Kurt Garloff] + - net: ubuntu focal prioritize netplan over eni even if both present + (#267) (LP: #1867029) + - cloudinit: refactor util.is_ipv4 to net.is_ipv4_address (#292) + - net/cmdline: replace type comments with annotations (#294) + - HACKING.rst: add Type Annotations design section (#293) + - net: introduce is_ip_address function (#288) + - CiTestCase: remove now-unneeded parse_and_read helper method (#286) + - .travis.yml: allow 30 minutes of inactivity in cloud tests (#287) + - sources/tests/test_init: drop use of deprecated inspect.getargspec (#285) + - setup.py: drop NIH check_output implementation (#282) + - Identify SAP Converged Cloud as OpenStack [Silvio Knizek] + - add Openbsd support (#147) [Gonéri Le Bouder] + - HACKING.rst: add examples of the two test class types (#278) + - VMWware: support to update guest info gc status if enabled (#261) + [xiaofengw-vmware] + - Add lp-to-git mapping for kgarloff (#279) + - set_passwords: avoid chpasswd on BSD (#268) [Gonéri Le Bouder] + - HACKING.rst: add Unit Testing design section (#277) + - util: read_cc_from_cmdline handle urlencoded yaml content (#275) + - distros/tests/test_init: add tests for _get_package_mirror_info (#272) + - HACKING.rst: add links to new Code Review Process doc (#276) + - freebsd: ensure package update works (#273) [Gonéri Le Bouder] + - doc: introduce Code Review Process documentation (#160) + - tools: use python3 (#274) + - cc_disk_setup: fix RuntimeError (#270) (LP: #1868327) + - cc_apt_configure/util: combine search_for_mirror implementations (#271) + - bsd: boottime does not depend on the libc soname (#269) + [Gonéri Le Bouder] + - test_oracle,DataSourceOracle: sort imports (#266) + - DataSourceOracle: update .network_config docstring (#257) + - cloudinit/tests: remove unneeded with_logs configuration (#263) + - .travis.yml: drop stale comment (#255) + - .gitignore: add more common directories (#258) + - ec2: render network on all NICs and add secondary IPs as static (#114) + (LP: #1866930) + - ec2 json validation: fix the reference to the 'merged_cfg' key (#256) + [Paride Legovini] + - releases.yaml: quote the Ubuntu version numbers (#254) [Paride Legovini] + - cloudinit: remove six from packaging/tooling (#253) + - util/netbsd: drop six usage (#252) + - workflows: introduce stale pull request workflow (#125) + - cc_resolv_conf: introduce tests and stabilise output across Python + versions (#251) + - fix minor issue with resolv_conf template (#144) [andreaf74] + - doc: CloudInit also support NetBSD (#250) [Gonéri Le Bouder] + - Add Netbsd support (#62) [Gonéri Le Bouder] + - tox.ini: avoid substition syntax that causes a traceback on xenial (#245) + - Add pub_key_ed25519 to cc_phone_home (#237) [Daniel Hensby] + - Introduce and use of a list of GitHub usernames that have signed CLA + (#244) + - workflows/cla.yml: use correct username for CLA check (#243) + - tox.ini: use xenial version of jsonpatch in CI (#242) + - workflows: CLA validation altered to fail status on pull_request (#164) + - tox.ini: bump pyflakes version to 2.1.1 (#239) + - cloudinit: move to pytest for running tests (#211) + - instance-data: add cloud-init merged_cfg and sys_info keys to json + (#214) (LP: #1865969) + - ec2: Do not fallback to IMDSv1 on EC2 (#216) + - instance-data: write redacted cfg to instance-data.json (#233) + (LP: #1865947) + - net: support network-config:disabled on the kernel commandline (#232) + (LP: #1862702) + - ec2: only redact token request headers in logs, avoid altering request + (#230) (LP: #1865882) + - docs: typo fixed: dta → data [Alexey Vazhnov] + - Fixes typo on Amazon Web Services (#217) [Nick Wales] + - Fix docs for OpenStack DMI Asset Tag (#228) + [Mark T. Voelker] (LP: #1669875) + - Add physical network type: cascading to openstack helpers (#200) + [sab-systems] + - tests: add focal integration tests for ubuntu (#225) + 20.1 - ec2: Do not log IMDSv2 token values, instead use REDACTED (#219) (LP: #1863943) @@ -96,7 +701,7 @@ - docs: add additional details to per-instance/once [Joshua Powers] - Update doc-requirements.txt [Joshua Powers] - doc-requirements: add missing dep [Joshua Powers] - - dhcp: Support RedHat dhcp rfc3442 lease format for option 121 (#76) + - dhcp: Support Red Hat dhcp rfc3442 lease format for option 121 (#76) [Eric Lafontaine] (LP: #1850642) - network_state: handle empty v1 config (#45) (LP: #1852496) - docs: Add document on how to report bugs [Joshua Powers] diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/analyze/dump.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/dump.py --- cloud-init-20.1-10-g71af48df/cloudinit/analyze/dump.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/dump.py 2021-05-11 16:34:29.000000000 +0000 @@ -4,6 +4,7 @@ from datetime import datetime import sys +from cloudinit import subp from cloudinit import util stage_to_description = { @@ -51,7 +52,7 @@ def parse_timestamp_from_date(timestampstr): - out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr]) + out, _ = subp.subp(['date', '+%s.%3N', '-d', timestampstr]) timestamp = out.strip() return float(timestamp) @@ -74,8 +75,12 @@ # # 2017-05-22 18:02:01,088 - util.py[DEBUG]: Cloud-init v. 0.7.9 running \ # 'init-local' at Mon, 22 May 2017 18:02:01 +0000. Up 2.0 seconds. + # + # Apr 30 19:39:11 cloud-init[2673]: handlers.py[DEBUG]: start: \ + # init-local/check-cache: attempting to read from cache [check] - separators = [' - ', ' [CLOUDINIT] '] + amazon_linux_2_sep = ' cloud-init[' + separators = [' - ', ' [CLOUDINIT] ', amazon_linux_2_sep] found = False for sep in separators: if sep in line: @@ -98,7 +103,14 @@ hostname = extra.split()[-1] else: hostname = timehost.split()[-1] - timestampstr = timehost.split(hostname)[0].strip() + if sep == amazon_linux_2_sep: + # This is an Amazon Linux style line, with no hostname and a PID. + # Use the whole of timehost as timestampstr, and strip off the PID + # from the start of eventstr. + timestampstr = timehost.strip() + eventstr = eventstr.split(maxsplit=1)[1] + else: + timestampstr = timehost.split(hostname)[0].strip() if 'Cloud-init v.' in eventstr: event_type = 'start' if 'running' in eventstr: diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/analyze/show.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/show.py --- cloud-init-20.1-10-g71af48df/cloudinit/analyze/show.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/show.py 2021-05-11 16:34:29.000000000 +0000 @@ -11,31 +11,29 @@ import time import sys +from cloudinit import subp from cloudinit import util from cloudinit.distros import uses_systemd -# An event: -''' -{ - "description": "executing late commands", - "event_type": "start", - "level": "INFO", - "name": "cmd-install/stage-late" - "origin": "cloudinit", - "timestamp": 1461164249.1590767, -}, - - { - "description": "executing late commands", - "event_type": "finish", - "level": "INFO", - "name": "cmd-install/stage-late", - "origin": "cloudinit", - "result": "SUCCESS", - "timestamp": 1461164249.1590767 - } +# Example events: +# { +# "description": "executing late commands", +# "event_type": "start", +# "level": "INFO", +# "name": "cmd-install/stage-late" +# "origin": "cloudinit", +# "timestamp": 1461164249.1590767, +# } +# { +# "description": "executing late commands", +# "event_type": "finish", +# "level": "INFO", +# "name": "cmd-install/stage-late", +# "origin": "cloudinit", +# "result": "SUCCESS", +# "timestamp": 1461164249.1590767 +# } -''' format_key = { '%d': 'delta', '%D': 'description', @@ -155,7 +153,7 @@ :return: whether the subp call failed or not ''' try: - value, err = util.subp(self.args, capture=True) + value, err = subp.subp(self.args, capture=True) if err: return err self.epoch = value @@ -215,7 +213,7 @@ with gather_timestamps_using_systemd ''' try: - data, _ = util.subp(['dmesg'], capture=True) + data, _ = subp.subp(['dmesg'], capture=True) split_entries = data[0].splitlines() for i in split_entries: if i.decode('UTF-8').find('user') != -1: @@ -269,7 +267,7 @@ except OSError as err: raise RuntimeError('Could not determine container boot ' 'time from /proc/1/cmdline. ({})' - .format(err)) + .format(err)) from err status = CONTAINER_CODE else: status = FAIL_CODE diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/analyze/tests/test_boot.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/tests/test_boot.py --- cloud-init-20.1-10-g71af48df/cloudinit/analyze/tests/test_boot.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/tests/test_boot.py 2021-05-11 16:34:29.000000000 +0000 @@ -25,7 +25,7 @@ m_get_linux_distro, m_is_FreeBSD): self.assertEqual(err_code, dist_check_timestamp()) - @mock.patch('cloudinit.util.subp', return_value=(0, 1)) + @mock.patch('cloudinit.subp.subp', return_value=(0, 1)) def test_subp_fails(self, m_subp): self.assertEqual(err_code, dist_check_timestamp()) @@ -42,7 +42,7 @@ with self.assertRaises(RuntimeError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_systemctl_works_correctly_threshold(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(1.0, reader.parse_epoch_as_float()) @@ -50,12 +50,12 @@ self.assertTrue(thresh < 1e-6) self.assertTrue(thresh > (-1 * 1e-6)) - @mock.patch('cloudinit.util.subp', return_value=('U=0', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=0', None)) def test_systemctl_succeed_zero(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(0.0, reader.parse_epoch_as_float()) - @mock.patch('cloudinit.util.subp', return_value=('U=1', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1', None)) def test_systemctl_succeed_distinct(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') val1 = reader.parse_epoch_as_float() @@ -64,13 +64,13 @@ val2 = reader2.parse_epoch_as_float() self.assertNotEqual(val1, val2) - @mock.patch('cloudinit.util.subp', return_value=('100', None)) + @mock.patch('cloudinit.subp.subp', return_value=('100', None)) def test_systemctl_epoch_not_splittable(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(IndexError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=foobar', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=foobar', None)) def test_systemctl_cannot_convert_epoch_to_float(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(ValueError): @@ -130,7 +130,7 @@ self.assertEqual(err_string, data) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_container_no_ci_log_line(self, m_is_container, m_subp): path = os.path.dirname(os.path.abspath(__file__)) log_path = path + '/boot-test.log' @@ -148,7 +148,7 @@ self.assertEqual(FAIL_CODE, finish_code) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) @mock.patch('cloudinit.analyze.__main__._get_events', return_value=[{ 'name': 'init-local', 'description': 'starting search', 'timestamp': 100000}]) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/analyze/tests/test_dump.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/tests/test_dump.py --- cloud-init-20.1-10-g71af48df/cloudinit/analyze/tests/test_dump.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/analyze/tests/test_dump.py 2021-05-11 16:34:29.000000000 +0000 @@ -5,7 +5,8 @@ from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) -from cloudinit.util import which, write_file +from cloudinit.util import write_file +from cloudinit.subp import which from cloudinit.tests.helpers import CiTestCase, mock, skipIf @@ -119,6 +120,23 @@ m_parse_from_date.assert_has_calls( [mock.call("2016-08-30 21:53:25.972325+00:00")]) + def test_parse_logline_returns_event_for_amazon_linux_2_line(self): + line = ( + "Apr 30 19:39:11 cloud-init[2673]: handlers.py[DEBUG]: start:" + " init-local/check-cache: attempting to read from cache [check]") + # Generate the expected value using `datetime`, so that TZ + # determination is consistent with the code under test. + timestamp_dt = datetime.strptime( + "Apr 30 19:39:11", "%b %d %H:%M:%S" + ).replace(year=datetime.now().year) + expected = { + 'description': 'attempting to read from cache [check]', + 'event_type': 'start', + 'name': 'init-local/check-cache', + 'origin': 'cloudinit', + 'timestamp': timestamp_dt.timestamp()} + self.assertEqual(expected, parse_ci_logline(line)) + SAMPLE_LOGS = dedent("""\ Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT] util.py[DEBUG]:\ diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/apport.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/apport.py --- cloud-init-20.1-10-g71af48df/cloudinit/apport.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/apport.py 2021-05-11 16:34:29.000000000 +0000 @@ -36,9 +36,12 @@ 'OVF', 'RbxCloud - (HyperOne, Rootbox, Rubikon)', 'OpenTelekomCloud', + 'SAP Converged Cloud', 'Scaleway', 'SmartOS', + 'UpCloud', 'VMware', + 'Vultr', 'ZStack', 'Other' ] diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/atomic_helper.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/atomic_helper.py --- cloud-init-20.1-10-g71af48df/cloudinit/atomic_helper.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/atomic_helper.py 2021-05-11 16:34:29.000000000 +0000 @@ -11,10 +11,10 @@ def write_file(filename, content, mode=_DEF_PERMS, - omode="wb", copy_mode=False): + omode="wb", preserve_mode=False): # open filename in mode 'omode', write content, set permissions to 'mode' - if copy_mode: + if preserve_mode: try: file_stat = os.stat(filename) mode = stat.S_IMODE(file_stat.st_mode) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/clean.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/clean.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/clean.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/clean.py 2021-05-11 16:34:29.000000000 +0000 @@ -10,9 +10,8 @@ import sys from cloudinit.stages import Init -from cloudinit.util import ( - ProcessExecutionError, del_dir, del_file, get_config_logfiles, - is_link, subp) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (del_dir, del_file, get_config_logfiles, is_link) def error(msg): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/logs.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/logs.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/logs.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/logs.py 2021-05-11 16:34:29.000000000 +0000 @@ -12,8 +12,8 @@ from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.temp_utils import tempdir -from cloudinit.util import ( - ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (chdir, copy, ensure_dir, write_file) CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/make_mime.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/make_mime.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/make_mime.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/make_mime.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,114 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Generate multi-part mime messages for user-data """ + +import argparse +import sys +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from cloudinit import log +from cloudinit.handlers import INCLUSION_TYPES_MAP +from . import addLogHandlerCLI + +NAME = 'make-mime' +LOG = log.getLogger(NAME) +EPILOG = ("Example: make-mime -a config.yaml:cloud-config " + "-a script.sh:x-shellscript > user-data") + + +def file_content_type(text): + """ Return file content type by reading the first line of the input. """ + try: + filename, content_type = text.split(":", 1) + return (open(filename, 'r'), filename, content_type.strip()) + except ValueError as e: + raise argparse.ArgumentError( + text, "Invalid value for %r" % (text) + ) from e + + +def get_parser(parser=None): + """Build or extend and arg parser for make-mime utility. + + @param parser: Optional existing ArgumentParser instance representing the + subcommand which will be extended to support the args of this utility. + + @returns: ArgumentParser with proper argument configuration. + """ + if not parser: + parser = argparse.ArgumentParser() + # update the parser's doc and add an epilog to show an example + parser.description = __doc__ + parser.epilog = EPILOG + parser.add_argument("-a", "--attach", dest="files", type=file_content_type, + action='append', default=[], + metavar=":", + help=("attach the given file as the specified " + "content-type")) + parser.add_argument('-l', '--list-types', action='store_true', + default=False, + help='List support cloud-init content types.') + parser.add_argument('-f', '--force', action='store_true', + default=False, + help='Ignore unknown content-type warnings') + return parser + + +def get_content_types(strip_prefix=False): + """ Return a list of cloud-init supported content types. Optionally + strip out the leading 'text/' of the type if strip_prefix=True. + """ + return sorted([ctype.replace("text/", "") if strip_prefix else ctype + for ctype in INCLUSION_TYPES_MAP.values()]) + + +def handle_args(name, args): + """Create a multi-part MIME archive for use as user-data. Optionally + print out the list of supported content types of cloud-init. + + Also setup CLI log handlers to report to stderr since this is a development + utility which should be run by a human on the CLI. + + @return 0 on success, 1 on failure. + """ + addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING) + if args.list_types: + print("\n".join(get_content_types(strip_prefix=True))) + return 0 + + sub_messages = [] + errors = [] + for i, (fh, filename, format_type) in enumerate(args.files): + contents = fh.read() + sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) + sub_message.add_header('Content-Disposition', + 'attachment; filename="%s"' % (filename)) + content_type = sub_message.get_content_type().lower() + if content_type not in get_content_types(): + level = "WARNING" if args.force else "ERROR" + msg = (level + ": content type %r for attachment %s " + "may be incorrect!") % (content_type, i + 1) + sys.stderr.write(msg + '\n') + errors.append(msg) + sub_messages.append(sub_message) + if len(errors) and not args.force: + sys.stderr.write("Invalid content-types, override with --force\n") + return 1 + combined_message = MIMEMultipart() + for msg in sub_messages: + combined_message.attach(msg) + print(combined_message) + return 0 + + +def main(): + args = get_parser().parse_args() + return(handle_args(NAME, args)) + + +if __name__ == '__main__': + sys.exit(main()) + + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/net_convert.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/net_convert.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/net_convert.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/net_convert.py 2021-05-11 16:34:29.000000000 +0000 @@ -28,11 +28,13 @@ if not parser: parser = argparse.ArgumentParser(prog=NAME, description=__doc__) parser.add_argument("-p", "--network-data", type=open, - metavar="PATH", required=True) + metavar="PATH", required=True, + help="The network configuration to read") parser.add_argument("-k", "--kind", choices=['eni', 'network_data.json', 'yaml', 'azure-imds', 'vmware-imc'], - required=True) + required=True, + help="The format of the given network config") parser.add_argument("-d", "--directory", metavar="PATH", help="directory to place output in", @@ -50,7 +52,8 @@ help='enable debug logging to stderr.') parser.add_argument("-O", "--output-kind", choices=['eni', 'netplan', 'sysconfig'], - required=True) + required=True, + help="The network config format to emit") return parser diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/parser.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/parser.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/parser.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/parser.py 2021-05-11 16:34:29.000000000 +0000 @@ -9,6 +9,7 @@ from . import net_convert from . import render +from . import make_mime def get_parser(parser=None): @@ -25,7 +26,9 @@ (net_convert.NAME, net_convert.__doc__, net_convert.get_parser, net_convert.handle_args), (render.NAME, render.__doc__, - render.get_parser, render.handle_args) + render.get_parser, render.handle_args), + (make_mime.NAME, make_mime.__doc__, + make_mime.get_parser, make_mime.handle_args), ] for (subcmd, helpmsg, get_parser, handler) in subcmds: parser = subparsers.add_parser(subcmd, help=helpmsg) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/render.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/render.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/render.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/render.py 2021-05-11 16:34:29.000000000 +0000 @@ -57,8 +57,9 @@ paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE) if not os.path.exists(instance_data_fn): LOG.warning( - 'Missing root-readable %s. Using redacted %s instead.', - instance_data_fn, redacted_data_fn) + 'Missing root-readable %s. Using redacted %s instead.', + instance_data_fn, redacted_data_fn + ) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/tests/test_logs.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/tests/test_logs.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/devel/tests/test_logs.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/devel/tests/test_logs.py 2021-05-11 16:34:29.000000000 +0000 @@ -8,7 +8,8 @@ from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.tests.helpers import ( FilesystemMockingTestCase, mock, wrap_and_call) -from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.subp import subp +from cloudinit.util import ensure_dir, load_file, write_file @mock.patch('cloudinit.cmd.devel.logs.os.getuid') diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/main.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/main.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/main.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/main.py 2021-05-11 16:34:29.000000000 +0000 @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/query.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/query.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/query.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/query.py 2021-05-11 16:34:29.000000000 +0000 @@ -1,6 +1,17 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""Query standardized instance metadata from the command line.""" +"""Query standardized instance metadata provided to machine, returning a JSON +structure. + +Some instance-data values may be binary on some platforms, such as userdata and +vendordata. Attempt to decompress and decode UTF-8 any binary values. + +Any binary values in the instance metadata will be base64-encoded and prefixed +with "ci-b64:" in the output. userdata and, where applicable, vendordata may +be provided to the machine gzip-compressed (and therefore as binary data). +query will attempt to decompress these to a string before emitting the JSON +output; if this fails, they are treated as binary. +""" import argparse from errno import EACCES @@ -30,7 +41,7 @@ """ if not parser: parser = argparse.ArgumentParser( - prog=NAME, description='Query cloud-init instance data') + prog=NAME, description=__doc__) parser.add_argument( '-d', '--debug', action='store_true', default=False, help='Add verbose messages during template render') @@ -52,8 +63,10 @@ ' /var/lib/cloud/instance/vendor-data.txt')) parser.add_argument( 'varname', type=str, nargs='?', - help=('A dot-delimited instance data variable to query from' - ' instance-data query. For example: v2.local_hostname')) + help=('A dot-delimited specific variable to query from' + ' instance-data. For example: v1.local_hostname. If the' + ' value is not JSON serializable, it will be base64-encoded and' + ' will contain the prefix "ci-b64:". ')) parser.add_argument( '-a', '--all', action='store_true', default=False, dest='dump_all', help='Dump all available instance-data') @@ -65,6 +78,21 @@ return parser +def load_userdata(ud_file_path): + """Attempt to return a string of user-data from ud_file_path + + Attempt to decode or decompress if needed. + If unable to decode the content, raw bytes will be returned. + + @returns: String of uncompressed userdata if possible, otherwise bytes. + """ + bdata = util.load_file(ud_file_path, decode=False) + try: + return bdata.decode('utf-8') + except UnicodeDecodeError: + return util.decomp_gzip(bdata, quiet=False, decode=True) + + def handle_args(name, args): """Handle calls to 'cloud-init query' as a subcommand.""" paths = None @@ -90,8 +118,9 @@ instance_data_fn = sensitive_data_fn else: LOG.warning( - 'Missing root-readable %s. Using redacted %s instead.', - sensitive_data_fn, redacted_data_fn) + 'Missing root-readable %s. Using redacted %s instead.', + sensitive_data_fn, redacted_data_fn + ) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn @@ -120,8 +149,8 @@ instance_data['vendordata'] = ( '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, vendor_data_fn)) else: - instance_data['userdata'] = util.load_file(user_data_fn) - instance_data['vendordata'] = util.load_file(vendor_data_fn) + instance_data['userdata'] = load_userdata(user_data_fn) + instance_data['vendordata'] = load_userdata(vendor_data_fn) if args.format: payload = '## template: jinja\n{fmt}'.format(fmt=args.format) rendered_payload = render_jinja_payload( diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_clean.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_clean.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_clean.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_clean.py 2021-05-11 16:34:29.000000000 +0000 @@ -167,7 +167,6 @@ wrap_and_call( 'cloudinit.cmd.clean', {'Init': {'side_effect': self.init_class}, - 'sys.exit': {'side_effect': self.sys_exit}, 'sys.argv': {'new': ['clean', '--logs']}}, clean.main) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_main.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_main.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_main.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_main.py 2021-05-11 16:34:29.000000000 +0000 @@ -18,8 +18,6 @@ class TestMain(FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestMain, self).setUp() self.new_root = self.tmp_dir() @@ -129,7 +127,8 @@ 'syslog_fix_perms': [ 'syslog:adm', 'root:adm', 'root:wheel', 'root:root' ], - 'vendor_data': {'enabled': True, 'prefix': []}}) + 'vendor_data': {'enabled': True, 'prefix': []}, + 'vendor_data2': {'enabled': True, 'prefix': []}}) updated_cfg.pop('system_info') self.assertEqual(updated_cfg, cfg) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_query.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_query.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_query.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_query.py 2021-05-11 16:34:29.000000000 +0000 @@ -1,195 +1,260 @@ # This file is part of cloud-init. See LICENSE file for license information. import errno -from io import StringIO +import gzip +from io import BytesIO +import json from textwrap import dedent -import os + +import pytest from collections import namedtuple from cloudinit.cmd import query from cloudinit.helpers import Paths from cloudinit.sources import ( REDACT_SENSITIVE_VALUE, INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE) -from cloudinit.tests.helpers import CiTestCase, mock -from cloudinit.util import ensure_dir, write_file +from cloudinit.tests.helpers import mock + +from cloudinit.util import b64e, write_file + +def _gzip_data(data): + with BytesIO() as iobuf: + with gzip.GzipFile(mode="wb", fileobj=iobuf) as gzfp: + gzfp.write(data) + return iobuf.getvalue() -class TestQuery(CiTestCase): - with_logs = True +@mock.patch("cloudinit.cmd.query.addLogHandlerCLI", lambda *args: "") +class TestQuery: args = namedtuple( 'queryargs', ('debug dump_all format instance_data list_keys user_data vendor_data' ' varname')) - def setUp(self): - super(TestQuery, self).setUp() - self.tmp = self.tmp_dir() - self.instance_data = self.tmp_path('instance-data', dir=self.tmp) + def _setup_paths(self, tmpdir, ud_val=None, vd_val=None): + """Write userdata and vendordata into a tmpdir. - def test_handle_args_error_on_missing_param(self): + Return: + 4-tuple : (paths, run_dir_path, userdata_path, vendordata_path) + """ + if ud_val: + user_data = tmpdir.join('user-data') + write_file(user_data.strpath, ud_val) + else: + user_data = None + if vd_val: + vendor_data = tmpdir.join('vendor-data') + write_file(vendor_data.strpath, vd_val) + else: + vendor_data = None + run_dir = tmpdir.join('run_dir') + run_dir.ensure_dir() + return ( + Paths({'run_dir': run_dir.strpath}), + run_dir, + user_data, + vendor_data + ) + + def test_handle_args_error_on_missing_param(self, caplog, capsys): """Error when missing required parameters and print usage.""" args = self.args( debug=False, dump_all=False, format=None, instance_data=None, list_keys=False, user_data=None, vendor_data=None, varname=None) - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(1, query.handle_args('anyname', args)) + with mock.patch( + "cloudinit.cmd.query.addLogHandlerCLI", return_value="" + ) as m_cli_log: + assert 1 == query.handle_args('anyname', args) expected_error = ( - 'ERROR: Expected one of the options: --all, --format, --list-keys' + 'Expected one of the options: --all, --format, --list-keys' ' or varname\n') - self.assertIn(expected_error, self.logs.getvalue()) - self.assertIn('usage: query', m_stdout.getvalue()) - self.assertIn(expected_error, m_stderr.getvalue()) + assert expected_error in caplog.text + out, _err = capsys.readouterr() + assert 'usage: query' in out + assert 1 == m_cli_log.call_count - def test_handle_args_error_on_missing_instance_data(self): + def test_handle_args_error_on_missing_instance_data(self, caplog, tmpdir): """When instance_data file path does not exist, log an error.""" - absent_fn = self.tmp_path('absent', dir=self.tmp) + absent_fn = tmpdir.join('absent') args = self.args( - debug=False, dump_all=True, format=None, instance_data=absent_fn, + debug=False, dump_all=True, format=None, + instance_data=absent_fn.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname=None) - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - self.assertEqual(1, query.handle_args('anyname', args)) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % absent_fn, - self.logs.getvalue()) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % absent_fn, - m_stderr.getvalue()) + assert 1 == query.handle_args('anyname', args) + + msg = 'Missing instance-data file: %s' % absent_fn + assert msg in caplog.text - def test_handle_args_error_when_no_read_permission_instance_data(self): + def test_handle_args_error_when_no_read_permission_instance_data( + self, caplog, tmpdir + ): """When instance_data file is unreadable, log an error.""" - noread_fn = self.tmp_path('unreadable', dir=self.tmp) - write_file(noread_fn, 'thou shall not pass') + noread_fn = tmpdir.join('unreadable') + noread_fn.write('thou shall not pass') args = self.args( - debug=False, dump_all=True, format=None, instance_data=noread_fn, + debug=False, dump_all=True, format=None, + instance_data=noread_fn.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname=None) - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with mock.patch('cloudinit.cmd.query.util.load_file') as m_load: - m_load.side_effect = OSError(errno.EACCES, 'Not allowed') - self.assertEqual(1, query.handle_args('anyname', args)) - self.assertIn( - "ERROR: No read permission on '%s'. Try sudo" % noread_fn, - self.logs.getvalue()) - self.assertIn( - "ERROR: No read permission on '%s'. Try sudo" % noread_fn, - m_stderr.getvalue()) + with mock.patch('cloudinit.cmd.query.util.load_file') as m_load: + m_load.side_effect = OSError(errno.EACCES, 'Not allowed') + assert 1 == query.handle_args('anyname', args) + msg = "No read permission on '%s'. Try sudo" % noread_fn + assert msg in caplog.text - def test_handle_args_defaults_instance_data(self): + def test_handle_args_defaults_instance_data(self, caplog, tmpdir): """When no instance_data argument, default to configured run_dir.""" args = self.args( debug=False, dump_all=True, format=None, instance_data=None, list_keys=False, user_data=None, vendor_data=None, varname=None) - run_dir = self.tmp_path('run_dir', dir=self.tmp) - ensure_dir(run_dir) - paths = Paths({'run_dir': run_dir}) - self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths') - self.m_paths.return_value = paths - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - self.assertEqual(1, query.handle_args('anyname', args)) - json_file = os.path.join(run_dir, INSTANCE_JSON_FILE) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % json_file, - self.logs.getvalue()) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % json_file, - m_stderr.getvalue()) + paths, run_dir, _, _ = self._setup_paths(tmpdir) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths + assert 1 == query.handle_args('anyname', args) + json_file = run_dir.join(INSTANCE_JSON_FILE) + msg = 'Missing instance-data file: %s' % json_file.strpath + assert msg in caplog.text - def test_handle_args_root_fallsback_to_instance_data(self): + def test_handle_args_root_fallsback_to_instance_data(self, caplog, tmpdir): """When no instance_data argument, root falls back to redacted json.""" args = self.args( debug=False, dump_all=True, format=None, instance_data=None, list_keys=False, user_data=None, vendor_data=None, varname=None) - run_dir = self.tmp_path('run_dir', dir=self.tmp) - ensure_dir(run_dir) - paths = Paths({'run_dir': run_dir}) - self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths') - self.m_paths.return_value = paths - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: + paths, run_dir, _, _ = self._setup_paths(tmpdir) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 0 + assert 1 == query.handle_args('anyname', args) + json_file = run_dir.join(INSTANCE_JSON_FILE) + sensitive_file = run_dir.join(INSTANCE_JSON_SENSITIVE_FILE) + msg = ( + 'Missing root-readable %s. Using redacted %s instead.' % + ( + sensitive_file.strpath, json_file.strpath + ) + ) + assert msg in caplog.text + + @pytest.mark.parametrize( + 'ud_src,ud_expected,vd_src,vd_expected', + ( + ('hi mom', 'hi mom', 'hi pops', 'hi pops'), + ('ud'.encode('utf-8'), 'ud', 'vd'.encode('utf-8'), 'vd'), + (_gzip_data(b'ud'), 'ud', _gzip_data(b'vd'), 'vd'), + (_gzip_data('ud'.encode('utf-8')), 'ud', _gzip_data(b'vd'), 'vd'), + ) + ) + def test_handle_args_root_processes_user_data( + self, ud_src, ud_expected, vd_src, vd_expected, capsys, tmpdir + ): + """Support reading multiple user-data file content types""" + paths, run_dir, user_data, vendor_data = self._setup_paths( + tmpdir, ud_val=ud_src, vd_val=vd_src + ) + sensitive_file = run_dir.join(INSTANCE_JSON_SENSITIVE_FILE) + sensitive_file.write('{"my-var": "it worked"}') + args = self.args( + debug=False, dump_all=True, format=None, instance_data=None, + list_keys=False, user_data=user_data.strpath, + vendor_data=vendor_data.strpath, varname=None) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths with mock.patch('os.getuid') as m_getuid: m_getuid.return_value = 0 - self.assertEqual(1, query.handle_args('anyname', args)) - json_file = os.path.join(run_dir, INSTANCE_JSON_FILE) - sensitive_file = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE) - self.assertIn( - 'WARNING: Missing root-readable %s. Using redacted %s instead.' % ( - sensitive_file, json_file), - m_stderr.getvalue()) - - def test_handle_args_root_uses_instance_sensitive_data(self): - """When no instance_data argument, root uses semsitive json.""" - user_data = self.tmp_path('user-data', dir=self.tmp) - vendor_data = self.tmp_path('vendor-data', dir=self.tmp) - write_file(user_data, 'ud') - write_file(vendor_data, 'vd') - run_dir = self.tmp_path('run_dir', dir=self.tmp) - sensitive_file = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE) - write_file(sensitive_file, '{"my-var": "it worked"}') - ensure_dir(run_dir) - paths = Paths({'run_dir': run_dir}) - self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths') - self.m_paths.return_value = paths + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + cmd_output = json.loads(out) + assert "it worked" == cmd_output['my_var'] + if ud_expected == "ci-b64:": + ud_expected = "ci-b64:{}".format(b64e(ud_src)) + if vd_expected == "ci-b64:": + vd_expected = "ci-b64:{}".format(b64e(vd_src)) + assert ud_expected == cmd_output['userdata'] + assert vd_expected == cmd_output['vendordata'] + + def test_handle_args_root_uses_instance_sensitive_data( + self, capsys, tmpdir + ): + """When no instance_data argument, root uses sensitive json.""" + paths, run_dir, user_data, vendor_data = self._setup_paths( + tmpdir, ud_val='ud', vd_val='vd' + ) + sensitive_file = run_dir.join(INSTANCE_JSON_SENSITIVE_FILE) + sensitive_file.write('{"my-var": "it worked"}') args = self.args( debug=False, dump_all=True, format=None, instance_data=None, - list_keys=False, user_data=vendor_data, vendor_data=vendor_data, - varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: + list_keys=False, user_data=user_data.strpath, + vendor_data=vendor_data.strpath, varname=None) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths with mock.patch('os.getuid') as m_getuid: m_getuid.return_value = 0 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual( - '{\n "my_var": "it worked",\n "userdata": "vd",\n ' - '"vendordata": "vd"\n}\n', m_stdout.getvalue()) + assert 0 == query.handle_args('anyname', args) + expected = ( + '{\n "my_var": "it worked",\n "userdata": "ud",\n ' + '"vendordata": "vd"\n}\n' + ) + out, _err = capsys.readouterr() + assert expected == out - def test_handle_args_dumps_all_instance_data(self): + def test_handle_args_dumps_all_instance_data(self, capsys, tmpdir): """When --all is specified query will dump all instance data vars.""" - write_file(self.instance_data, '{"my-var": "it worked"}') + instance_data = tmpdir.join('instance-data') + instance_data.write('{"my-var": "it worked"}') args = self.args( debug=False, dump_all=True, format=None, - instance_data=self.instance_data, list_keys=False, + instance_data=instance_data.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual( + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + expected = ( '{\n "my_var": "it worked",\n "userdata": "<%s> file:ud",\n' ' "vendordata": "<%s> file:vd"\n}\n' % ( - REDACT_SENSITIVE_VALUE, REDACT_SENSITIVE_VALUE), - m_stdout.getvalue()) + REDACT_SENSITIVE_VALUE, REDACT_SENSITIVE_VALUE + ) + ) + out, _err = capsys.readouterr() + assert expected == out - def test_handle_args_returns_top_level_varname(self): + def test_handle_args_returns_top_level_varname(self, capsys, tmpdir): """When the argument varname is passed, report its value.""" - write_file(self.instance_data, '{"my-var": "it worked"}') + instance_data = tmpdir.join('instance-data') + instance_data.write('{"my-var": "it worked"}') args = self.args( debug=False, dump_all=True, format=None, - instance_data=self.instance_data, list_keys=False, + instance_data=instance_data.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname='my_var') - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual('it worked\n', m_stdout.getvalue()) + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert 'it worked\n' == out - def test_handle_args_returns_nested_varname(self): + def test_handle_args_returns_nested_varname(self, capsys, tmpdir): """If user_data file is a jinja template render instance-data vars.""" - write_file(self.instance_data, - '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}') + instance_data = tmpdir.join('instance-data') + instance_data.write( + '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}' + ) args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, user_data='ud', vendor_data='vd', - list_keys=False, varname='v1.key_2') - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual('value-2\n', m_stdout.getvalue()) - - def test_handle_args_returns_standardized_vars_to_top_level_aliases(self): + instance_data=instance_data.strpath, user_data='ud', + vendor_data='vd', list_keys=False, varname='v1.key_2') + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert 'value-2\n' == out + + def test_handle_args_returns_standardized_vars_to_top_level_aliases( + self, capsys, tmpdir + ): """Any standardized vars under v# are promoted as top-level aliases.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},' ' "top": "gun"}') expected = dedent("""\ @@ -209,65 +274,68 @@ """) args = self.args( debug=False, dump_all=True, format=None, - instance_data=self.instance_data, user_data='ud', vendor_data='vd', - list_keys=False, varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual(expected, m_stdout.getvalue()) - - def test_handle_args_list_keys_sorts_top_level_keys_when_no_varname(self): + instance_data=instance_data.strpath, user_data='ud', + vendor_data='vd', list_keys=False, varname=None) + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert expected == out + + def test_handle_args_list_keys_sorts_top_level_keys_when_no_varname( + self, capsys, tmpdir + ): """Sort all top-level keys when only --list-keys provided.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},' ' "top": "gun"}') expected = 'top\nuserdata\nv1\nv1_1\nv2\nv2_2\nvendordata\n' args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, list_keys=True, user_data='ud', - vendor_data='vd', varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual(expected, m_stdout.getvalue()) - - def test_handle_args_list_keys_sorts_nested_keys_when_varname(self): + instance_data=instance_data.strpath, list_keys=True, + user_data='ud', vendor_data='vd', varname=None) + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert expected == out + + def test_handle_args_list_keys_sorts_nested_keys_when_varname( + self, capsys, tmpdir + ): """Sort all nested keys of varname object when --list-keys provided.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2":' + ' {"v2_2": "val2.2"}, "top": "gun"}') expected = 'v1_1\nv1_2\n' args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, list_keys=True, + instance_data=instance_data.strpath, list_keys=True, user_data='ud', vendor_data='vd', varname='v1') - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual(expected, m_stdout.getvalue()) - - def test_handle_args_list_keys_errors_when_varname_is_not_a_dict(self): + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert expected == out + + def test_handle_args_list_keys_errors_when_varname_is_not_a_dict( + self, caplog, tmpdir + ): """Raise an error when --list-keys and varname specify a non-list.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2": ' + '{"v2_2": "val2.2"}, "top": "gun"}') - expected_error = "ERROR: --list-keys provided but 'top' is not a dict" + expected_error = "--list-keys provided but 'top' is not a dict" args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, list_keys=True, user_data='ud', - vendor_data='vd', varname='top') - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(1, query.handle_args('anyname', args)) - self.assertEqual('', m_stdout.getvalue()) - self.assertIn(expected_error, m_stderr.getvalue()) + instance_data=instance_data.strpath, list_keys=True, + user_data='ud', vendor_data='vd', varname='top') + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 1 == query.handle_args('anyname', args) + assert expected_error in caplog.text # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_status.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_status.py --- cloud-init-20.1-10-g71af48df/cloudinit/cmd/tests/test_status.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/cmd/tests/test_status.py 2021-05-11 16:34:29.000000000 +0000 @@ -382,7 +382,6 @@ wrap_and_call( 'cloudinit.cmd.status', {'sys.argv': {'new': ['status']}, - 'sys.exit': {'side_effect': self.sys_exit}, '_is_cloudinit_disabled': (False, ''), 'Init': {'side_effect': self.init_class}}, status.main) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_apk_configure.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_apk_configure.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_apk_configure.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_apk_configure.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,263 @@ +# Copyright (c) 2020 Dermot Bradley +# +# Author: Dermot Bradley +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Apk Configure: Configures apk repositories file.""" + +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit import temp_utils +from cloudinit import templater +from cloudinit import util +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +# If no mirror is specified then use this one +DEFAULT_MIRROR = "https://alpine.global.ssl.fastly.net/alpine" + + +REPOSITORIES_TEMPLATE = """\ +## template:jinja +# +# Created by cloud-init +# +# This file is written on first boot of an instance +# + +{{ alpine_baseurl }}/{{ alpine_version }}/main +{% if community_enabled -%} +{{ alpine_baseurl }}/{{ alpine_version }}/community +{% endif -%} +{% if testing_enabled -%} +{% if alpine_version != 'edge' %} +# +# Testing - using with non-Edge installation may cause problems! +# +{% endif %} +{{ alpine_baseurl }}/edge/testing +{% endif %} +{% if local_repo != '' %} + +# +# Local repo +# +{{ local_repo }}/{{ alpine_version }} +{% endif %} + +""" + + +frequency = PER_INSTANCE +distros = ['alpine'] +schema = { + 'id': 'cc_apk_configure', + 'name': 'APK Configure', + 'title': 'Configure apk repositories file', + 'description': dedent("""\ + This module handles configuration of the /etc/apk/repositories file. + + .. note:: + To ensure that apk configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + """), + 'distros': distros, + 'examples': [ + dedent("""\ + # Keep the existing /etc/apk/repositories file unaltered. + apk_repos: + preserve_repositories: true + """), + dedent("""\ + # Create repositories file for Alpine v3.12 main and community + # using default mirror site. + apk_repos: + alpine_repo: + community_enabled: true + version: 'v3.12' + """), + dedent("""\ + # Create repositories file for Alpine Edge main, community, and + # testing using a specified mirror site and also a local repo. + apk_repos: + alpine_repo: + base_url: 'https://some-alpine-mirror/alpine' + community_enabled: true + testing_enabled: true + version: 'edge' + local_repo_base_url: 'https://my-local-server/local-alpine' + """), + ], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'apk_repos': { + 'type': 'object', + 'properties': { + 'preserve_repositories': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + By default, cloud-init will generate a new repositories + file ``/etc/apk/repositories`` based on any valid + configuration settings specified within a apk_repos + section of cloud config. To disable this behavior and + preserve the repositories file from the pristine image, + set ``preserve_repositories`` to ``true``. + + The ``preserve_repositories`` option overrides + all other config keys that would alter + ``/etc/apk/repositories``. + """) + }, + 'alpine_repo': { + 'type': ['object', 'null'], + 'properties': { + 'base_url': { + 'type': 'string', + 'default': DEFAULT_MIRROR, + 'description': dedent("""\ + The base URL of an Alpine repository, or + mirror, to download official packages from. + If not specified then it defaults to ``{}`` + """.format(DEFAULT_MIRROR)) + }, + 'community_enabled': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + Whether to add the Community repo to the + repositories file. By default the Community + repo is not included. + """) + }, + 'testing_enabled': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + Whether to add the Testing repo to the + repositories file. By default the Testing + repo is not included. It is only recommended + to use the Testing repo on a machine running + the ``Edge`` version of Alpine as packages + installed from Testing may have dependancies + that conflict with those in non-Edge Main or + Community repos." + """) + }, + 'version': { + 'type': 'string', + 'description': dedent("""\ + The Alpine version to use (e.g. ``v3.12`` or + ``edge``) + """) + }, + }, + 'required': ['version'], + 'minProperties': 1, + 'additionalProperties': False, + }, + 'local_repo_base_url': { + 'type': 'string', + 'description': dedent("""\ + The base URL of an Alpine repository containing + unofficial packages + """) + } + }, + 'required': [], + 'minProperties': 1, # Either preserve_repositories or alpine_repo + 'additionalProperties': False, + } + } +} + +__doc__ = get_schema_doc(schema) + + +def handle(name, cfg, cloud, log, _args): + """ + Call to handle apk_repos sections in cloud-config file. + + @param name: The module name "apk-configure" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The CloudInit object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param _args: Any module arguments from cloud.cfg + """ + + # If there is no "apk_repos" section in the configuration + # then do nothing. + apk_section = cfg.get('apk_repos') + if not apk_section: + LOG.debug(("Skipping module named %s," + " no 'apk_repos' section found"), name) + return + + validate_cloudconfig_schema(cfg, schema) + + # If "preserve_repositories" is explicitly set to True in + # the configuration do nothing. + if util.get_cfg_option_bool(apk_section, 'preserve_repositories', False): + LOG.debug(("Skipping module named %s," + " 'preserve_repositories' is set"), name) + return + + # If there is no "alpine_repo" subsection of "apk_repos" present in the + # configuration then do nothing, as at least "version" is required to + # create valid repositories entries. + alpine_repo = apk_section.get('alpine_repo') + if not alpine_repo: + LOG.debug(("Skipping module named %s," + " no 'alpine_repo' configuration found"), name) + return + + # If there is no "version" value present in configuration then do nothing. + alpine_version = alpine_repo.get('version') + if not alpine_version: + LOG.debug(("Skipping module named %s," + " 'version' not specified in alpine_repo"), name) + return + + local_repo = apk_section.get('local_repo_base_url', '') + + _write_repositories_file(alpine_repo, alpine_version, local_repo) + + +def _write_repositories_file(alpine_repo, alpine_version, local_repo): + """ + Write the /etc/apk/repositories file with the specified entries. + + @param alpine_repo: A nested dict of the alpine_repo configuration. + @param alpine_version: A string of the Alpine version to use. + @param local_repo: A string containing the base URL of a local repo. + """ + + repo_file = '/etc/apk/repositories' + + alpine_baseurl = alpine_repo.get('base_url', DEFAULT_MIRROR) + + params = {'alpine_baseurl': alpine_baseurl, + 'alpine_version': alpine_version, + 'community_enabled': alpine_repo.get('community_enabled'), + 'testing_enabled': alpine_repo.get('testing_enabled'), + 'local_repo': local_repo} + + tfile = temp_utils.mkstemp(prefix='template_name-', suffix=".tmpl") + template_fn = tfile[1] # Filepath is second item in tuple + util.write_file(template_fn, content=REPOSITORIES_TEMPLATE) + + LOG.debug('Generating Alpine repository configuration file: %s', + repo_file) + templater.render_to_file(template_fn, repo_file, params) + # Clean up temporary template + util.del_file(template_fn) + + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_apt_configure.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_apt_configure.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_apt_configure.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_apt_configure.py 2021-05-11 16:34:29.000000000 +0000 @@ -6,228 +6,372 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Apt Configure -------------- -**Summary:** configure apt - -This module handles both configuration of apt options and adding source lists. -There are configuration options such as ``apt_get_wrapper`` and -``apt_get_command`` that control how cloud-init invokes apt-get. -These configuration options are handled on a per-distro basis, so consult -documentation for cloud-init's distro support for instructions on using -these config options. - -.. note:: - To ensure that apt configuration is valid yaml, any strings containing - special characters, especially ``:`` should be quoted. - -.. note:: - For more information about apt configuration, see the - ``Additional apt configuration`` example. - -**Preserve sources.list:** - -By default, cloud-init will generate a new sources list in -``/etc/apt/sources.list.d`` based on any changes specified in cloud config. -To disable this behavior and preserve the sources list from the pristine image, -set ``preserve_sources_list`` to ``true``. - -.. note:: - The ``preserve_sources_list`` option overrides all other config keys that - would alter ``sources.list`` or ``sources.list.d``, **except** for - additional sources to be added to ``sources.list.d``. - -**Disable source suites:** - -Entries in the sources list can be disabled using ``disable_suites``, which -takes a list of suites to be disabled. If the string ``$RELEASE`` is present in -a suite in the ``disable_suites`` list, it will be replaced with the release -name. If a suite specified in ``disable_suites`` is not present in -``sources.list`` it will be ignored. For convenience, several aliases are -provided for ``disable_suites``: - - - ``updates`` => ``$RELEASE-updates`` - - ``backports`` => ``$RELEASE-backports`` - - ``security`` => ``$RELEASE-security`` - - ``proposed`` => ``$RELEASE-proposed`` - - ``release`` => ``$RELEASE`` - -.. note:: - When a suite is disabled using ``disable_suites``, its entry in - ``sources.list`` is not deleted; it is just commented out. - -**Configure primary and security mirrors:** - -The primary and security archive mirrors can be specified using the ``primary`` -and ``security`` keys, respectively. Both the ``primary`` and ``security`` keys -take a list of configs, allowing mirrors to be specified on a per-architecture -basis. Each config is a dictionary which must have an entry for ``arches``, -specifying which architectures that config entry is for. The keyword -``default`` applies to any architecture not explicitly listed. The mirror url -can be specified with the ``uri`` key, or a list of mirrors to check can be -provided in order, with the first mirror that can be resolved being selected. -This allows the same configuration to be used in different environment, with -different hosts used for a local apt mirror. If no mirror is provided by -``uri`` or ``search``, ``search_dns`` may be used to search for dns names in -the format ``-mirror`` in each of the following: - - - fqdn of this host per cloud metadata - - localdomain - - domains listed in ``/etc/resolv.conf`` - -If there is a dns entry for ``-mirror``, then it is assumed that there -is a distro mirror at ``http://-mirror./``. If the -``primary`` key is defined, but not the ``security`` key, then then -configuration for ``primary`` is also used for ``security``. If ``search_dns`` -is used for the ``security`` key, the search pattern will be. -``-security-mirror``. - -If no mirrors are specified, or all lookups fail, then default mirrors defined -in the datasource are used. If none are present in the datasource either the -following defaults are used: - - - primary: ``http://archive.ubuntu.com/ubuntu`` - - security: ``http://security.ubuntu.com/ubuntu`` - -**Specify sources.list template:** - -A custom template for rendering ``sources.list`` can be specefied with -``sources_list``. If no ``sources_list`` template is given, cloud-init will -use sane default. Within this template, the following strings will be replaced -with the appropriate values: - - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY`` - -**Pass configuration to apt:** - -Apt configuration can be specified using ``conf``. Configuration is specified -as a string. For multiline apt configuration, make sure to follow yaml syntax. - -**Configure apt proxy:** - -Proxy configuration for apt can be specified using ``conf``, but proxy config -keys also exist for convenience. The proxy config keys, ``http_proxy``, -``ftp_proxy``, and ``https_proxy`` may be used to specify a proxy for http, ftp -and https protocols respectively. The ``proxy`` key also exists as an alias for -``http_proxy``. Proxy url is specified in the format -``://[[user][:pass]@]host[:port]/``. - -**Add apt repos by regex:** - -All source entries in ``apt-sources`` that match regex in -``add_apt_repo_match`` will be added to the system using -``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it defaults -to ``^[\\w-]+:\\w`` - -**Add source list entries:** - -Source list entries can be specified as a dictionary under the ``sources`` -config key, with key in the dict representing a different source file. The key -of each source entry will be used as an id that can be referenced in -other config entries, as well as the filename for the source's configuration -under ``/etc/apt/sources.list.d``. If the name does not end with ``.list``, -it will be appended. If there is no configuration for a key in ``sources``, no -file will be written, but the key may still be referred to as an id in other -``sources`` entries. - -Each entry under ``sources`` is a dictionary which may contain any of the -following optional keys: - - - ``source``: a sources.list entry (some variable replacements apply) - - ``keyid``: a key to import via shortid or fingerprint - - ``key``: a raw PGP key - - ``keyserver``: alternate keyserver to pull ``keyid`` key from - -The ``source`` key supports variable replacements for the following strings: - - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE`` - -**Internal name:** ``cc_apt_configure`` - -**Module frequency:** per instance - -**Supported distros:** ubuntu, debian - -**Config keys**:: - - apt: - preserve_sources_list: - disable_suites: +"""Apt Configure: Configure apt for the user.""" + +import glob +import os +import re +from textwrap import dedent + +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit import gpg +from cloudinit import log as logging +from cloudinit import subp +from cloudinit import templater +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + +# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + +frequency = PER_INSTANCE +distros = ["ubuntu", "debian"] +mirror_property = { + 'type': 'array', + 'item': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['arches'], + 'properties': { + 'arches': { + 'type': 'array', + 'item': { + 'type': 'string' + }, + 'minItems': 1 + }, + 'uri': { + 'type': 'string', + 'format': 'uri' + }, + 'search': { + 'type': 'array', + 'item': { + 'type': 'string', + 'format': 'uri' + }, + 'minItems': 1 + }, + 'search_dns': { + 'type': 'boolean', + } + } + } +} +schema = { + 'id': 'cc_apt_configure', + 'name': 'Apt Configure', + 'title': 'Configure apt for the user', + 'description': dedent("""\ + This module handles both configuration of apt options and adding + source lists. There are configuration options such as + ``apt_get_wrapper`` and ``apt_get_command`` that control how + cloud-init invokes apt-get. These configuration options are + handled on a per-distro basis, so consult documentation for + cloud-init's distro support for instructions on using + these config options. + + .. note:: + To ensure that apt configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + + .. note:: + For more information about apt configuration, see the + ``Additional apt configuration`` example."""), + 'distros': distros, + 'examples': [dedent("""\ + apt: + preserve_sources_list: false + disable_suites: - $RELEASE-updates - backports - $RELEASE - mysuite - primary: + primary: - arches: - amd64 - i386 - default - uri: "http://us.archive.ubuntu.com/ubuntu" + uri: 'http://us.archive.ubuntu.com/ubuntu' search: - - "http://cool.but-sometimes-unreachable.com/ubuntu" - - "http://us.archive.ubuntu.com/ubuntu" + - 'http://cool.but-sometimes-unreachable.com/ubuntu' + - 'http://us.archive.ubuntu.com/ubuntu' search_dns: - arches: - s390x - arm64 - uri: "http://archive-to-use-for-arm64.example.com/ubuntu" - security: + uri: 'http://archive-to-use-for-arm64.example.com/ubuntu' + security: - arches: - default search_dns: true - sources_list: | - deb $MIRROR $RELEASE main restricted - deb-src $MIRROR $RELEASE main restricted - deb $PRIMARY $RELEASE universe restricted - deb $SECURITY $RELEASE-security multiverse - debconf_selections: - set1: the-package the-package/some-flag boolean true - conf: | - APT { - Get { - Assume-Yes "true"; - Fix-Broken "true"; + sources_list: | + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + debconf_selections: + set1: the-package the-package/some-flag boolean true + conf: | + APT { + Get { + Assume-Yes 'true'; + Fix-Broken 'true'; + } + } + proxy: 'http://[[user][:pass]@]host[:port]/' + http_proxy: 'http://[[user][:pass]@]host[:port]/' + ftp_proxy: 'ftp://[[user][:pass]@]host[:port]/' + https_proxy: 'https://[[user][:pass]@]host[:port]/' + sources: + source1: + keyid: 'keyid' + keyserver: 'keyserverurl' + source: 'deb http:/// xenial main' + source2: + source: 'ppa:' + source3: + source: 'deb $MIRROR $RELEASE multiverse' + key: | + ------BEGIN PGP PUBLIC KEY BLOCK------- + + ------END PGP PUBLIC KEY BLOCK-------""")], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'apt': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'preserve_sources_list': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + By default, cloud-init will generate a new sources + list in ``/etc/apt/sources.list.d`` based on any + changes specified in cloud config. To disable this + behavior and preserve the sources list from the + pristine image, set ``preserve_sources_list`` + to ``true``. + + The ``preserve_sources_list`` option overrides + all other config keys that would alter + ``sources.list`` or ``sources.list.d``, + **except** for additional sources to be added + to ``sources.list.d``.""") + }, + 'disable_suites': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'uniqueItems': True, + 'description': dedent("""\ + Entries in the sources list can be disabled using + ``disable_suites``, which takes a list of suites + to be disabled. If the string ``$RELEASE`` is + present in a suite in the ``disable_suites`` list, + it will be replaced with the release name. If a + suite specified in ``disable_suites`` is not + present in ``sources.list`` it will be ignored. + For convenience, several aliases are provided for + ``disable_suites``: + + - ``updates`` => ``$RELEASE-updates`` + - ``backports`` => ``$RELEASE-backports`` + - ``security`` => ``$RELEASE-security`` + - ``proposed`` => ``$RELEASE-proposed`` + - ``release`` => ``$RELEASE``. + + When a suite is disabled using ``disable_suites``, + its entry in ``sources.list`` is not deleted; it + is just commented out.""") + }, + 'primary': { + **mirror_property, + 'description': dedent("""\ + The primary and security archive mirrors can + be specified using the ``primary`` and + ``security`` keys, respectively. Both the + ``primary`` and ``security`` keys take a list + of configs, allowing mirrors to be specified + on a per-architecture basis. Each config is a + dictionary which must have an entry for + ``arches``, specifying which architectures + that config entry is for. The keyword + ``default`` applies to any architecture not + explicitly listed. The mirror url can be specified + with the ``uri`` key, or a list of mirrors to + check can be provided in order, with the first + mirror that can be resolved being selected. This + allows the same configuration to be used in + different environment, with different hosts used + for a local apt mirror. If no mirror is provided + by ``uri`` or ``search``, ``search_dns`` may be + used to search for dns names in the format + ``-mirror`` in each of the following: + + - fqdn of this host per cloud metadata, + - localdomain, + - domains listed in ``/etc/resolv.conf``. + + If there is a dns entry for ``-mirror``, + then it is assumed that there is a distro mirror + at ``http://-mirror./``. + If the ``primary`` key is defined, but not the + ``security`` key, then then configuration for + ``primary`` is also used for ``security``. + If ``search_dns`` is used for the ``security`` + key, the search pattern will be + ``-security-mirror``. + + If no mirrors are specified, or all lookups fail, + then default mirrors defined in the datasource + are used. If none are present in the datasource + either the following defaults are used: + + - ``primary`` => \ + ``http://archive.ubuntu.com/ubuntu``. + - ``security`` => \ + ``http://security.ubuntu.com/ubuntu`` + """)}, + 'security': { + **mirror_property, + 'description': dedent("""\ + Please refer to the primary config documentation""") + }, + 'add_apt_repo_match': { + 'type': 'string', + 'default': ADD_APT_REPO_MATCH, + 'description': dedent("""\ + All source entries in ``apt-sources`` that match + regex in ``add_apt_repo_match`` will be added to + the system using ``add-apt-repository``. If + ``add_apt_repo_match`` is not specified, it + defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) + }, + 'debconf_selections': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Debconf additional configurations can be specified as a + dictionary under the ``debconf_selections`` config + key, with each key in the dict representing a + different set of configurations. The value of each key + must be a string containing all the debconf + configurations that must be applied. We will bundle + all of the values and pass them to + ``debconf-set-selections``. Therefore, each value line + must be a valid entry for ``debconf-set-selections``, + meaning that they must possess for distinct fields: + + ``pkgname question type answer`` + + Where: + + - ``pkgname`` is the name of the package. + - ``question`` the name of the questions. + - ``type`` is the type of question. + - ``answer`` is the value used to ansert the \ + question. + + For example: \ + ``ippackage ippackage/ip string 127.0.01`` + """) + }, + 'sources_list': { + 'type': 'string', + 'description': dedent("""\ + Specifies a custom template for rendering + ``sources.list`` . If no ``sources_list`` template + is given, cloud-init will use sane default. Within + this template, the following strings will be + replaced with the appropriate values: + + - ``$MIRROR`` + - ``$RELEASE`` + - ``$PRIMARY`` + - ``$SECURITY``""") + }, + 'conf': { + 'type': 'string', + 'description': dedent("""\ + Specify configuration for apt, such as proxy + configuration. This configuration is specified as a + string. For multiline apt configuration, make sure + to follow yaml syntax.""") + }, + 'https_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify https apt proxy. + https proxy url is specified in the format + ``https://[[user][:pass]@]host[:port]/``.""") + }, + 'http_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify http apt proxy. + http proxy url is specified in the format + ``http://[[user][:pass]@]host[:port]/``.""") + }, + 'proxy': { + 'type': 'string', + 'description': 'Alias for defining a http apt proxy.' + }, + 'ftp_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify ftp apt proxy. + ftp proxy url is specified in the format + ``ftp://[[user][:pass]@]host[:port]/``.""") + }, + 'sources': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Source list entries can be specified as a + dictionary under the ``sources`` config key, with + each key in the dict representing a different source + file. The key of each source entry will be used + as an id that can be referenced in other config + entries, as well as the filename for the source's + configuration under ``/etc/apt/sources.list.d``. + If the name does not end with ``.list``, it will + be appended. If there is no configuration for a + key in ``sources``, no file will be written, but + the key may still be referred to as an id in other + ``sources`` entries. + + Each entry under ``sources`` is a dictionary which + may contain any of the following optional keys: + + - ``source``: a sources.list entry \ + (some variable replacements apply). + - ``keyid``: a key to import via shortid or \ + fingerprint. + - ``key``: a raw PGP key. + - ``keyserver``: alternate keyserver to pull \ + ``keyid`` key from. + + The ``source`` key supports variable + replacements for the following strings: + + - ``$MIRROR`` + - ``$PRIMARY`` + - ``$SECURITY`` + - ``$RELEASE``""") } } - proxy: "http://[[user][:pass]@]host[:port]/" - http_proxy: "http://[[user][:pass]@]host[:port]/" - ftp_proxy: "ftp://[[user][:pass]@]host[:port]/" - https_proxy: "https://[[user][:pass]@]host[:port]/" - sources: - source1: - keyid: "keyid" - keyserver: "keyserverurl" - source: "deb http:/// xenial main" - source2: - source: "ppa:" - source3: - source: "deb $MIRROR $RELEASE multiverse" - key: | - ------BEGIN PGP PUBLIC KEY BLOCK------- - - ------END PGP PUBLIC KEY BLOCK------- -""" + } + } +} -import glob -import os -import re +__doc__ = get_schema_doc(schema) -from cloudinit import gpg -from cloudinit import log as logging -from cloudinit import templater -from cloudinit import util - -LOG = logging.getLogger(__name__) - -# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') -ADD_APT_REPO_MATCH = r"^[\w-]+:\w" # place where apt stores cached repository data APT_LISTS = "/var/lib/apt/lists" @@ -245,7 +389,7 @@ PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} PRIMARY_ARCHES = ['amd64', 'i386'] -PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el'] +PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el', 'riscv64'] def get_default_mirrors(arch=None, target=None): @@ -279,6 +423,7 @@ "Expected dictionary for 'apt' config, found {config_type}".format( config_type=type(cfg))) + validate_cloudconfig_schema(cfg, schema) apply_debconf_selections(cfg, target) apply_apt(cfg, cloud, target) @@ -287,7 +432,7 @@ # if no config was provided, should apt configuration be done? if util.system_is_snappy(): return False, "system is snappy." - if not (util.which('apt-get') or util.which('apt')): + if not (subp.which('apt-get') or subp.which('apt')): return False, "no apt commands." return True, "Apt is available." @@ -334,7 +479,7 @@ def debconf_set_selections(selections, target=None): if not selections.endswith(b'\n'): selections += b'\n' - util.subp(['debconf-set-selections'], data=selections, target=target, + subp.subp(['debconf-set-selections'], data=selections, target=target, capture=True) @@ -359,7 +504,7 @@ "but cannot be unconfigured: %s", unhandled) if len(to_config): - util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + + subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + list(to_config), data=None, target=target, capture=True) @@ -402,7 +547,7 @@ def clean_cloud_init(target): """clean out any local cloud-init config""" flist = glob.glob( - util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) LOG.debug("cleaning cloud-init config from: %s", flist) for dpkg_cfg in flist: @@ -431,7 +576,7 @@ """rename_apt_lists - rename apt lists to preserve old cache data""" default_mirrors = get_default_mirrors(arch) - pre = util.target_path(target, APT_LISTS) + pre = subp.target_path(target, APT_LISTS) for (name, omirror) in default_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: @@ -550,8 +695,8 @@ """ LOG.debug("Adding key:\n'%s'", key) try: - util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) - except util.ProcessExecutionError: + subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) + except subp.ProcessExecutionError: LOG.exception("failed to add apt GPG Key to apt keyring") raise @@ -614,13 +759,13 @@ if aa_repo_match(source): try: - util.subp(["add-apt-repository", source], target=target) - except util.ProcessExecutionError: + subp.subp(["add-apt-repository", source], target=target) + except subp.ProcessExecutionError: LOG.exception("add-apt-repository failed.") raise continue - sourcefn = util.target_path(target, ent['filename']) + sourcefn = subp.target_path(target, ent['filename']) try: contents = "%s\n" % (source) util.write_file(sourcefn, contents, omode="a") @@ -763,25 +908,6 @@ return cfg -def search_for_mirror(candidates): - """ - Search through a list of mirror urls for one that works - This needs to return quickly. - """ - if candidates is None: - return None - - LOG.debug("search for mirror in candidates: '%s'", candidates) - for cand in candidates: - try: - if util.is_resolvable_url(cand): - LOG.debug("found working mirror: '%s'", cand) - return cand - except Exception: - pass - return None - - def search_for_mirror_dns(configured, mirrortype, cfg, cloud): """ Try to resolve a list of predefines DNS names to pick mirrors @@ -813,7 +939,7 @@ for post in doms: mirror_list.append(mirrorfmt % (post)) - mirror = search_for_mirror(mirror_list) + mirror = util.search_for_mirror(mirror_list) return mirror @@ -876,7 +1002,7 @@ # fallback to search if specified if mirror is None: # list of mirrors to try to resolve - mirror = search_for_mirror(mcfg.get("search", None)) + mirror = util.search_for_mirror(mcfg.get("search", None)) # fallback to search_dns if specified if mirror is None: diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_apt_pipelining.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_apt_pipelining.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_apt_pipelining.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_apt_pipelining.py 2021-05-11 16:34:29.000000000 +0000 @@ -9,7 +9,7 @@ -------------- **Summary:** configure apt pipelining -This module configures apt's ``Acquite::http::Pipeline-Depth`` option, whcih +This module configures apt's ``Acquite::http::Pipeline-Depth`` option, which controls how apt handles HTTP pipelining. It may be useful for pipelining to be disabled, because some web servers, such as S3 do not pipeline properly (LP: #948461). The ``apt_pipelining`` config key may be set to ``false`` to disable diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_bootcmd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_bootcmd.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_bootcmd.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_bootcmd.py 2021-05-11 16:34:29.000000000 +0000 @@ -16,6 +16,7 @@ get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -99,7 +100,7 @@ if iid: env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] - util.subp(cmd, env=env, capture=False) + subp.subp(cmd, env=env, capture=False) except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_byobu.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_byobu.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_byobu.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_byobu.py 2021-05-11 16:34:29.000000000 +0000 @@ -39,6 +39,7 @@ """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util distros = ['ubuntu', 'debian'] @@ -93,6 +94,6 @@ if len(shcmd): cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] log.debug("Setting byobu to %s", value) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ca_certs.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ca_certs.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ca_certs.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ca_certs.py 2021-05-11 16:34:29.000000000 +0000 @@ -16,11 +16,16 @@ certificates must be specified using valid yaml. in order to specify a multiline certificate, the yaml multiline list syntax must be used +.. note:: + For Alpine Linux the "remove-defaults" functionality works if the + ca-certificates package is installed but not if the + ca-certificates-bundle package is installed. + **Internal name:** ``cc_ca_certs`` **Module frequency:** per instance -**Supported distros:** ubuntu, debian +**Supported distros:** alpine, debian, ubuntu, rhel **Config keys**:: @@ -36,59 +41,113 @@ import os +from cloudinit import subp from cloudinit import util -CA_CERT_PATH = "/usr/share/ca-certificates/" -CA_CERT_FILENAME = "cloud-init-ca-certs.crt" -CA_CERT_CONFIG = "/etc/ca-certificates.conf" -CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" -CA_CERT_FULL_PATH = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - -distros = ['ubuntu', 'debian'] +DEFAULT_CONFIG = { + 'ca_cert_path': '/usr/share/ca-certificates/', + 'ca_cert_filename': 'cloud-init-ca-certs.crt', + 'ca_cert_config': '/etc/ca-certificates.conf', + 'ca_cert_system_path': '/etc/ssl/certs/', + 'ca_cert_update_cmd': ['update-ca-certificates'] +} +DISTRO_OVERRIDES = { + 'rhel': { + 'ca_cert_path': '/usr/share/pki/ca-trust-source/', + 'ca_cert_filename': 'anchors/cloud-init-ca-certs.crt', + 'ca_cert_config': None, + 'ca_cert_system_path': '/etc/pki/ca-trust/', + 'ca_cert_update_cmd': ['update-ca-trust'] + } +} + + +distros = ['alpine', 'debian', 'ubuntu', 'rhel'] + + +def _distro_ca_certs_configs(distro_name): + """Return a distro-specific ca_certs config dictionary + + @param distro_name: String providing the distro class name. + @returns: Dict of distro configurations for ca-cert. + """ + cfg = DISTRO_OVERRIDES.get(distro_name, DEFAULT_CONFIG) + cfg['ca_cert_full_path'] = os.path.join(cfg['ca_cert_path'], + cfg['ca_cert_filename']) + return cfg -def update_ca_certs(): +def update_ca_certs(distro_cfg): """ Updates the CA certificate cache on the current machine. + + @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - util.subp(["update-ca-certificates"], capture=False) + subp.subp(distro_cfg['ca_cert_update_cmd'], capture=False) -def add_ca_certs(certs): +def add_ca_certs(distro_cfg, certs): """ Adds certificates to the system. To actually apply the new certificates you must also call L{update_ca_certs}. + @param distro_cfg: A hash providing _distro_ca_certs_configs function. @param certs: A list of certificate strings. """ - if certs: - # First ensure they are strings... - cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) + if not certs: + return + # First ensure they are strings... + cert_file_contents = "\n".join([str(c) for c in certs]) + util.write_file(distro_cfg['ca_cert_full_path'], + cert_file_contents, + mode=0o644) + update_cert_config(distro_cfg) + +def update_cert_config(distro_cfg): + """ + Update Certificate config file to add the file path managed cloud-init + + @param distro_cfg: A hash providing _distro_ca_certs_configs function. + """ + if distro_cfg['ca_cert_config'] is None: + return + if os.stat(distro_cfg['ca_cert_config']).st_size == 0: + # If the CA_CERT_CONFIG file is empty (i.e. all existing + # CA certs have been deleted) then simply output a single + # line with the cloud-init cert filename. + out = "%s\n" % distro_cfg['ca_cert_filename'] + else: # Append cert filename to CA_CERT_CONFIG file. # We have to strip the content because blank lines in the file # causes subsequent entries to be ignored. (LP: #1077020) - orig = util.load_file(CA_CERT_CONFIG) - cur_cont = '\n'.join([line for line in orig.splitlines() - if line != CA_CERT_FILENAME]) - out = "%s\n%s\n" % (cur_cont.rstrip(), CA_CERT_FILENAME) - util.write_file(CA_CERT_CONFIG, out, omode="wb") + orig = util.load_file(distro_cfg['ca_cert_config']) + cr_cont = '\n'.join([line for line in orig.splitlines() + if line != distro_cfg['ca_cert_filename']]) + out = "%s\n%s\n" % (cr_cont.rstrip(), + distro_cfg['ca_cert_filename']) + util.write_file(distro_cfg['ca_cert_config'], out, omode="wb") -def remove_default_ca_certs(): +def remove_default_ca_certs(distro_name, distro_cfg): """ Removes all default trusted CA certificates from the system. To actually apply the change you must also call L{update_ca_certs}. + + @param distro_name: String providing the distro class name. + @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - util.delete_dir_contents(CA_CERT_PATH) - util.delete_dir_contents(CA_CERT_SYSTEM_PATH) - util.write_file(CA_CERT_CONFIG, "", mode=0o644) - debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - util.subp(('debconf-set-selections', '-'), debconf_sel) + util.delete_dir_contents(distro_cfg['ca_cert_path']) + util.delete_dir_contents(distro_cfg['ca_cert_system_path']) + util.write_file(distro_cfg['ca_cert_config'], "", mode=0o644) + + if distro_name in ['debian', 'ubuntu']: + debconf_sel = ( + "ca-certificates ca-certificates/trust_new_crts " + "select no") + subp.subp(('debconf-set-selections', '-'), debconf_sel) -def handle(name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud, log, _args): """ Call to handle ca-cert sections in cloud-config file. @@ -105,22 +164,23 @@ return ca_cert_cfg = cfg['ca-certs'] + distro_cfg = _distro_ca_certs_configs(cloud.distro.name) # If there is a remove-defaults option set to true, remove the system # default trusted CA certs first. if ca_cert_cfg.get("remove-defaults", False): log.debug("Removing default certificates") - remove_default_ca_certs() + remove_default_ca_certs(cloud.distro.name, distro_cfg) # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted") if trusted_certs: log.debug("Adding %d certificates" % len(trusted_certs)) - add_ca_certs(trusted_certs) + add_ca_certs(distro_cfg, trusted_certs) # Update the system with the new cert configuration. log.debug("Updating certificates") - update_ca_certs() + update_ca_certs(distro_cfg) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_chef.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_chef.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_chef.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_chef.py 2021-05-11 16:34:29.000000000 +0000 @@ -6,78 +6,22 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Chef ----- -**Summary:** module that configures, starts and installs chef. - -This module enables chef to be installed (from packages or -from gems, or from omnibus). Before this occurs chef configurations are -written to disk (validation.pem, client.pem, firstboot.json, client.rb), -and needed chef folders/directories are created (/etc/chef and /var/log/chef -and so-on). Then once installing proceeds correctly if configured chef will -be started (in daemon mode or in non-daemon mode) and then once that has -finished (if ran in non-daemon mode this will be when chef finishes -converging, if ran in daemon mode then no further actions are possible since -chef will have forked into its own process) then a post run function can -run that can do finishing activities (such as removing the validation pem -file). - -**Internal name:** ``cc_chef`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - chef: - directories: (defaulting to /etc/chef, /var/log/chef, /var/lib/chef, - /var/cache/chef, /var/backups/chef, /var/run/chef) - validation_cert: (optional string to be written to file validation_key) - special value 'system' means set use existing file - validation_key: (optional the path for validation_cert. default - /etc/chef/validation.pem) - firstboot_path: (path to write run_list and initial_attributes keys that - should also be present in this configuration, defaults - to /etc/chef/firstboot.json) - exec: boolean to run or not run chef (defaults to false, unless - a gem installed is requested - where this will then default - to true) - - chef.rb template keys (if falsey, then will be skipped and not - written to /etc/chef/client.rb) - - chef: - client_key: - encrypted_data_bag_secret: - environment: - file_backup_path: - file_cache_path: - json_attribs: - log_level: - log_location: - node_name: - omnibus_url: - omnibus_url_retries: - omnibus_version: - pid_file: - server_url: - show_time: - ssl_verify_mode: - validation_cert: - validation_key: - validation_name: -""" +"""Chef: module that configures, starts and installs chef.""" import itertools import json import os +from textwrap import dedent +from cloudinit import subp +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit import templater +from cloudinit import temp_utils from cloudinit import url_helper from cloudinit import util +from cloudinit.settings import PER_ALWAYS + RUBY_VERSION_DEFAULT = "1.8" @@ -98,6 +42,8 @@ OMNIBUS_URL_RETRIES = 5 CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' +CHEF_ENCRYPTED_DATA_BAG_PATH = '/etc/chef/encrypted_data_bag_secret' +CHEF_ENVIRONMENT = '_default' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... @@ -107,11 +53,11 @@ 'log_location': '/var/log/chef/client.log', 'validation_key': CHEF_VALIDATION_PEM_PATH, 'validation_cert': None, - 'client_key': "/etc/chef/client.pem", + 'client_key': '/etc/chef/client.pem', 'json_attribs': CHEF_FB_PATH, - 'file_cache_path': "/var/cache/chef", - 'file_backup_path': "/var/backups/chef", - 'pid_file': "/var/run/chef/client.pid", + 'file_cache_path': '/var/cache/chef', + 'file_backup_path': '/var/backups/chef', + 'pid_file': '/var/run/chef/client.pid', 'show_time': True, 'encrypted_data_bag_secret': None, } @@ -122,7 +68,6 @@ 'client_key', 'file_cache_path', 'json_attribs', - 'file_cache_path', 'pid_file', 'encrypted_data_bag_secret', ]) @@ -134,6 +79,7 @@ 'node_name', 'environment', 'validation_name', + 'chef_license', ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' @@ -141,12 +87,277 @@ CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) -def is_installed(): - if not os.path.isfile(CHEF_EXEC_PATH): - return False - if not os.access(CHEF_EXEC_PATH, os.X_OK): - return False - return True +frequency = PER_ALWAYS +distros = ["all"] +schema = { + 'id': 'cc_chef', + 'name': 'Chef', + 'title': 'module that configures, starts and installs chef', + 'description': dedent("""\ + This module enables chef to be installed (from packages, + gems, or from omnibus). Before this occurs, chef configuration is + written to disk (validation.pem, client.pem, firstboot.json, + client.rb), and required directories are created (/etc/chef and + /var/log/chef and so-on). If configured, chef will be + installed and started in either daemon or non-daemon mode. + If run in non-daemon mode, post run actions are executed to do + finishing activities such as removing validation.pem."""), + 'distros': distros, + 'examples': [dedent(""" + chef: + directories: + - /etc/chef + - /var/log/chef + validation_cert: system + install_type: omnibus + initial_attributes: + apache: + prefork: + maxclients: 100 + keepalive: off + run_list: + - recipe[apache2] + - role[db] + encrypted_data_bag_secret: /etc/chef/encrypted_data_bag_secret + environment: _default + log_level: :auto + omnibus_url_retries: 2 + server_url: https://chef.yourorg.com:4000 + ssl_verify_mode: :verify_peer + validation_name: yourorg-validator""")], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'chef': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'directories': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'uniqueItems': True, + 'description': dedent("""\ + Create the necessary directories for chef to run. By + default, it creates the following directories: + + {chef_dirs}""").format( + chef_dirs="\n".join( + [" - ``{}``".format(d) for d in CHEF_DIRS] + ) + ) + }, + 'validation_cert': { + 'type': 'string', + 'description': dedent("""\ + Optional string to be written to file validation_key. + Special value ``system`` means set use existing file. + """) + }, + 'validation_key': { + 'type': 'string', + 'default': CHEF_VALIDATION_PEM_PATH, + 'description': dedent("""\ + Optional path for validation_cert. default to + ``{}``.""".format(CHEF_VALIDATION_PEM_PATH)) + }, + 'firstboot_path': { + 'type': 'string', + 'default': CHEF_FB_PATH, + 'description': dedent("""\ + Path to write run_list and initial_attributes keys that + should also be present in this configuration, defaults + to ``{}``.""".format(CHEF_FB_PATH)) + }, + 'exec': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + define if we should run or not run chef (defaults to + false, unless a gem installed is requested where this + will then default to true).""") + }, + 'client_key': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['client_key'], + 'description': dedent("""\ + Optional path for client_cert. default to + ``{}``.""".format(CHEF_RB_TPL_DEFAULTS['client_key'])) + }, + 'encrypted_data_bag_secret': { + 'type': 'string', + 'default': None, + 'description': dedent("""\ + Specifies the location of the secret key used by chef + to encrypt data items. By default, this path is set + to None, meaning that chef will have to look at the + path ``{}`` for it. + """.format(CHEF_ENCRYPTED_DATA_BAG_PATH)) + }, + 'environment': { + 'type': 'string', + 'default': CHEF_ENVIRONMENT, + 'description': dedent("""\ + Specifies which environment chef will use. By default, + it will use the ``{}`` configuration. + """.format(CHEF_ENVIRONMENT)) + }, + 'file_backup_path': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['file_backup_path'], + 'description': dedent("""\ + Specifies the location in which backup files are + stored. By default, it uses the + ``{}`` location.""".format( + CHEF_RB_TPL_DEFAULTS['file_backup_path'])) + }, + 'file_cache_path': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['file_cache_path'], + 'description': dedent("""\ + Specifies the location in which chef cache files will + be saved. By default, it uses the ``{}`` + location.""".format( + CHEF_RB_TPL_DEFAULTS['file_cache_path'])) + }, + 'json_attribs': { + 'type': 'string', + 'default': CHEF_FB_PATH, + 'description': dedent("""\ + Specifies the location in which some chef json data is + stored. By default, it uses the + ``{}`` location.""".format(CHEF_FB_PATH)) + }, + 'log_level': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['log_level'], + 'description': dedent("""\ + Defines the level of logging to be stored in the log + file. By default this value is set to ``{}``. + """.format(CHEF_RB_TPL_DEFAULTS['log_level'])) + }, + 'log_location': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['log_location'], + 'description': dedent("""\ + Specifies the location of the chef lof file. By + default, the location is specified at + ``{}``.""".format( + CHEF_RB_TPL_DEFAULTS['log_location'])) + }, + 'node_name': { + 'type': 'string', + 'description': dedent("""\ + The name of the node to run. By default, we will + use th instance id as the node name.""") + }, + 'omnibus_url': { + 'type': 'string', + 'default': OMNIBUS_URL, + 'description': dedent("""\ + Omnibus URL if chef should be installed through + Omnibus. By default, it uses the + ``{}``.""".format(OMNIBUS_URL)) + }, + 'omnibus_url_retries': { + 'type': 'integer', + 'default': OMNIBUS_URL_RETRIES, + 'description': dedent("""\ + The number of retries that will be attempted to reach + the Omnibus URL""") + }, + 'omnibus_version': { + 'type': 'string', + 'description': dedent("""\ + Optional version string to require for omnibus + install.""") + }, + 'pid_file': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['pid_file'], + 'description': dedent("""\ + The location in which a process identification + number (pid) is saved. By default, it saves + in the ``{}`` location.""".format( + CHEF_RB_TPL_DEFAULTS['pid_file'])) + }, + 'server_url': { + 'type': 'string', + 'description': 'The URL for the chef server' + }, + 'show_time': { + 'type': 'boolean', + 'default': True, + 'description': 'Show time in chef logs' + }, + 'ssl_verify_mode': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['ssl_verify_mode'], + 'description': dedent("""\ + Set the verify mode for HTTPS requests. We can have + two possible values for this parameter: + + - ``:verify_none``: No validation of SSL \ + certificates. + - ``:verify_peer``: Validate all SSL certificates. + + By default, the parameter is set as ``{}``. + """.format(CHEF_RB_TPL_DEFAULTS['ssl_verify_mode'])) + }, + 'validation_name': { + 'type': 'string', + 'description': dedent("""\ + The name of the chef-validator key that Chef Infra + Client uses to access the Chef Infra Server during + the initial Chef Infra Client run.""") + }, + 'force_install': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + If set to ``True``, forces chef installation, even + if it is already installed.""") + }, + 'initial_attributes': { + 'type': 'object', + 'items': { + 'type': 'string' + }, + 'description': dedent("""\ + Specify a list of initial attributes used by the + cookbooks.""") + }, + 'install_type': { + 'type': 'string', + 'default': 'packages', + 'description': dedent("""\ + The type of installation for chef. It can be one of + the following values: + + - ``packages`` + - ``gems`` + - ``omnibus``""") + }, + 'run_list': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'description': 'A run list for a first boot json.' + }, + "chef_license": { + 'type': 'string', + 'description': dedent("""\ + string that indicates if user accepts or not license + related to some of chef products""") + } + } + } + } +} + +__doc__ = get_schema_doc(schema) def post_run_chef(chef_cfg, log): @@ -196,6 +407,8 @@ log.debug(("Skipping module named %s," " no 'chef' key in configuration"), name) return + + validate_cloudconfig_schema(cfg, schema) chef_cfg = cfg['chef'] # Ensure the chef directories we use exist @@ -223,7 +436,7 @@ iid = str(cloud.datasource.get_instance_id()) params = get_template_params(iid, chef_cfg, log) # Do a best effort attempt to ensure that the template values that - # are associated with paths have there parent directory created + # are associated with paths have their parent directory created # before they are used by the chef-client itself. param_paths = set() for (k, v) in params.items(): @@ -253,9 +466,10 @@ # Try to install chef, if its not already installed... force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) - if not is_installed() or force_install: + installed = subp.is_exe(CHEF_EXEC_PATH) + if not installed or force_install: run = install_chef(cloud, chef_cfg, log) - elif is_installed(): + elif installed: run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) else: run = False @@ -280,7 +494,32 @@ cmd.extend(CHEF_EXEC_DEF_ARGS) else: cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) + + +def subp_blob_in_tempfile(blob, *args, **kwargs): + """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. + + 'basename' as a kwarg allows providing the basename for the file. + The 'args' argument to subp will be updated with the full path to the + filename as the first argument. + """ + basename = kwargs.pop('basename', "subp_blob") + + if len(args) == 0 and 'args' not in kwargs: + args = [tuple()] + + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + with temp_utils.tempdir(needs_exe=True) as tmpd: + tmpf = os.path.join(tmpd, basename) + if 'args' in kwargs: + kwargs['args'] = [tmpf] + list(kwargs['args']) + else: + args = list(args) + args[0] = [tmpf] + args[0] + + util.write_file(tmpf, blob, mode=0o700) + return subp.subp(*args, **kwargs) def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): @@ -303,7 +542,7 @@ else: args = ['-v', omnibus_version] content = url_helper.readurl(url=url, retries=retries).contents - return util.subp_blob_in_tempfile( + return subp_blob_in_tempfile( blob=content, args=args, basename='chef-omnibus-install', capture=False) @@ -352,11 +591,11 @@ if not os.path.exists('/usr/bin/ruby'): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '-v %s' % chef_version, '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_disable_ec2_metadata.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_disable_ec2_metadata.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_disable_ec2_metadata.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_disable_ec2_metadata.py 2021-05-11 16:34:29.000000000 +0000 @@ -26,6 +26,7 @@ disable_ec2_metadata: """ +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -40,15 +41,15 @@ disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) if disabled: reject_cmd = None - if util.which('ip'): + if subp.which('ip'): reject_cmd = REJECT_CMD_IP - elif util.which('ifconfig'): + elif subp.which('ifconfig'): reject_cmd = REJECT_CMD_IF else: log.error(('Neither "route" nor "ip" command found, unable to ' 'manipulate routing table')) return - util.subp(reject_cmd, capture=False) + subp.subp(reject_cmd, capture=False) else: log.debug(("Skipping module named %s," " disabling the ec2 route not enabled"), name) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_disk_setup.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_disk_setup.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_disk_setup.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_disk_setup.py 2021-05-11 16:34:29.000000000 +0000 @@ -35,7 +35,7 @@ partition type. The size for partitions is specified in **percentage** of disk space, not in bytes (e.g. a size of 33 would take up 1/3 of the disk space). The ``overwrite`` option controls whether this module tries to be safe about -writing partition talbes or not. If ``overwrite: false`` is set, the device +writing partition tables or not. If ``overwrite: false`` is set, the device will be checked for a partition table and for a file system and if either is found, the operation will be skipped. If ``overwrite: true`` is set, no checks will be performed. @@ -99,6 +99,7 @@ from cloudinit.settings import PER_INSTANCE from cloudinit import util +from cloudinit import subp import logging import os import shlex @@ -106,13 +107,13 @@ frequency = PER_INSTANCE # Define the commands to use -UDEVADM_CMD = util.which('udevadm') -SFDISK_CMD = util.which("sfdisk") -SGDISK_CMD = util.which("sgdisk") -LSBLK_CMD = util.which("lsblk") -BLKID_CMD = util.which("blkid") -BLKDEV_CMD = util.which("blockdev") -WIPEFS_CMD = util.which("wipefs") +SFDISK_CMD = subp.which("sfdisk") +SGDISK_CMD = subp.which("sgdisk") +LSBLK_CMD = subp.which("lsblk") +BLKID_CMD = subp.which("blkid") +BLKDEV_CMD = subp.which("blockdev") +PARTPROBE_CMD = subp.which("partprobe") +WIPEFS_CMD = subp.which("wipefs") LANG_C_ENV = {'LANG': 'C'} @@ -163,7 +164,7 @@ def update_disk_setup_devices(disk_setup, tformer): # update 'disk_setup' dictionary anywhere were a device may occur # update it with the response from 'tformer' - for origname in disk_setup.keys(): + for origname in list(disk_setup): transformed = tformer(origname) if transformed is None or transformed == origname: continue @@ -248,9 +249,11 @@ info = None try: - info, _err = util.subp(lsblk_cmd) + info, _err = subp.subp(lsblk_cmd) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] @@ -310,9 +313,11 @@ blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device] try: - out, _err = util.subp(blkid_cmd, rcs=[0, 2]) + out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e if out: if len(out.splitlines()) == 1: @@ -427,16 +432,16 @@ else: return globals()[func_name] - except KeyError: - raise Exception("No such function %s to call!" % func_name) + except KeyError as e: + raise Exception("No such function %s to call!" % func_name) from e def get_hdd_size(device): try: - size_in_bytes, _ = util.subp([BLKDEV_CMD, '--getsize64', device]) - sector_size, _ = util.subp([BLKDEV_CMD, '--getss', device]) + size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) + sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) + raise Exception("Failed to get %s size\n%s" % (device, e)) from e return int(size_in_bytes) / int(sector_size) @@ -452,10 +457,11 @@ read_parttbl(device) prt_cmd = [SFDISK_CMD, "-l", device] try: - out, _err = util.subp(prt_cmd, data="%s\n" % layout) + out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e found_layout = [] for line in out.splitlines(): @@ -482,10 +488,11 @@ def check_partition_gpt_layout(device, layout): prt_cmd = [SGDISK_CMD, '-p', device] try: - out, _err = util.subp(prt_cmd, update_env=LANG_C_ENV) + out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e out_lines = iter(out.splitlines()) # Skip header. Output looks like: @@ -655,9 +662,11 @@ wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] try: LOG.info("Purging filesystem on /dev/%s", d['name']) - util.subp(wipefs_cmd) - except Exception: - raise Exception("Failed FS purge of /dev/%s" % d['name']) + subp.subp(wipefs_cmd) + except Exception as e: + raise Exception( + "Failed FS purge of /dev/%s" % d['name'] + ) from e purge_disk_ptable(device) @@ -676,13 +685,16 @@ def read_parttbl(device): """ - Use partprobe instead of 'udevadm'. Partprobe is the only - reliable way to probe the partition table. + `Partprobe` is preferred over `blkdev` since it is more reliably + able to probe the partition table. """ - blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device] + if PARTPROBE_CMD is not None: + probe_cmd = [PARTPROBE_CMD, device] + else: + probe_cmd = [BLKDEV_CMD, '--rereadpt', device] util.udevadm_settle() try: - util.subp(blkdev_cmd) + subp.subp(probe_cmd) except Exception as e: util.logexc(LOG, "Failed reading the partition table %s" % e) @@ -697,25 +709,27 @@ # Create the partitions prt_cmd = [SFDISK_CMD, "--Linux", "--unit=S", "--force", device] try: - util.subp(prt_cmd, data="%s\n" % layout) + subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Failed to partition device %s\n%s" % (device, e)) + raise Exception( + "Failed to partition device %s\n%s" % (device, e) + ) from e read_parttbl(device) def exec_mkpart_gpt(device, layout): try: - util.subp([SGDISK_CMD, '-Z', device]) + subp.subp([SGDISK_CMD, '-Z', device]) for index, (partition_type, (start, end)) in enumerate(layout): index += 1 - util.subp([SGDISK_CMD, + subp.subp([SGDISK_CMD, '-n', '{}:{}:{}'.format(index, start, end), device]) if partition_type is not None: # convert to a 4 char (or more) string right padded with 0 # 82 -> 8200. 'Linux' -> 'Linux' pinput = str(partition_type).ljust(4, "0") - util.subp( + subp.subp( [SGDISK_CMD, '-t', '{}:{}'.format(index, pinput), device]) except Exception: LOG.warning("Failed to partition device %s", device) @@ -967,9 +981,9 @@ fs_cmd) else: # Find the mkfs command - mkfs_cmd = util.which("mkfs.%s" % fs_type) + mkfs_cmd = subp.which("mkfs.%s" % fs_type) if not mkfs_cmd: - mkfs_cmd = util.which("mk%s" % fs_type) + mkfs_cmd = subp.which("mk%s" % fs_type) if not mkfs_cmd: LOG.warning("Cannot create fstype '%s'. No mkfs.%s command", @@ -994,8 +1008,8 @@ LOG.debug("Creating file system %s on %s", label, device) LOG.debug(" Using cmd: %s", str(fs_cmd)) try: - util.subp(fs_cmd, shell=shell) + subp.subp(fs_cmd, shell=shell) except Exception as e: - raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) + raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_emit_upstart.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_emit_upstart.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_emit_upstart.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_emit_upstart.py 2021-05-11 16:34:29.000000000 +0000 @@ -25,7 +25,7 @@ from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS -from cloudinit import util +from cloudinit import subp frequency = PER_ALWAYS @@ -43,9 +43,9 @@ del myenv['UPSTART_SESSION'] check_cmd = ['initctl', 'version'] try: - (out, _err) = util.subp(check_cmd, env=myenv) + (out, _err) = subp.subp(check_cmd, env=myenv) return 'upstart' in out - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.debug("'%s' returned '%s', not using upstart", ' '.join(check_cmd), e.exit_code) return False @@ -66,7 +66,7 @@ for n in event_names: cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: # TODO(harlowja), use log exception from utils?? log.warning("Emission of upstart event %s failed due to: %s", n, e) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_fan.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_fan.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_fan.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_fan.py 2021-05-11 16:34:29.000000000 +0000 @@ -39,6 +39,7 @@ from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -62,8 +63,8 @@ def run(cmd, msg): try: - return util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + return subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: LOG.warning("failed: %s (%s): %s", service, cmd, e) return False @@ -94,7 +95,7 @@ util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") distro = cloud.distro - if not util.which('fanctl'): + if not subp.which('fanctl'): distro.install_packages(['ubuntu-fan']) stop_update_start( diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_final_message.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_final_message.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_final_message.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_final_message.py 2021-05-11 16:34:29.000000000 +0000 @@ -78,7 +78,7 @@ boot_fin_fn = cloud.paths.boot_finished try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) - util.write_file(boot_fin_fn, contents) + util.write_file(boot_fin_fn, contents, ensure_dir_exists=False) except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_growpart.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_growpart.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_growpart.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_growpart.py 2021-05-11 16:34:29.000000000 +0000 @@ -16,12 +16,13 @@ than the pristine image uses, as it allows the instance to automatically make use of the extra space. -The devices run growpart on are specified as a list under the ``devices`` key. -Each entry in the devices list can be either the path to the device's -mountpoint in the filesystem or a path to the block device in ``/dev``. +The devices on which to run growpart are specified as a list under the +``devices`` key. Each entry in the devices list can be either the path to the +device's mountpoint in the filesystem or a path to the block device in +``/dev``. The utility to use for resizing can be selected using the ``mode`` config key. -If ``mode`` key is set to ``auto``, then any available utility (either +If the ``mode`` key is set to ``auto``, then any available utility (either ``growpart`` or BSD ``gpart``) will be used. If neither utility is available, no error will be raised. If ``mode`` is set to ``growpart``, then the ``growpart`` utility will be used. If this utility is not available on the @@ -34,7 +35,7 @@ configuration for both should work for most cloud instances. To explicitly prevent ``cloud-initramfs-tools`` from running ``growroot``, the file ``/etc/growroot-disabled`` can be created. By default, both ``growroot`` and -``cc_growpart`` will check for the existance of this file and will not run if +``cc_growpart`` will check for the existence of this file and will not run if it is present. However, this file can be ignored for ``cc_growpart`` by setting ``ignore_growroot_disabled`` to ``true``. For more information on ``cloud-initramfs-tools`` see: https://launchpad.net/cloud-initramfs-tools @@ -70,6 +71,7 @@ from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -131,30 +133,30 @@ myenv['LANG'] = 'C' try: - (out, _err) = util.subp(["growpart", "--help"], env=myenv) + (out, _err) = subp.subp(["growpart", "--help"], env=myenv) if re.search(r"--update\s+", out): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False def resize(self, diskdev, partnum, partdev): before = get_size(partdev) try: - util.subp(["growpart", '--dry-run', diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", '--dry-run', diskdev, partnum]) + except subp.ProcessExecutionError as e: if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, before) try: - util.subp(["growpart", diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", diskdev, partnum]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, get_size(partdev)) @@ -165,11 +167,11 @@ myenv['LANG'] = 'C' try: - (_out, err) = util.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) + (_out, err) = subp.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) if re.search(r"gpart recover ", err): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False @@ -182,22 +184,18 @@ be recovered. """ try: - util.subp(["gpart", "recover", diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "recover", diskdev]) + except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e before = get_size(partdev) try: - util.subp(["gpart", "resize", "-i", partnum, diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "resize", "-i", partnum, diskdev]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) - raise ResizeFailedException(e) - - # Since growing the FS requires a reboot, make sure we reboot - # first when this module has finished. - open('/var/run/reboot-required', 'a').close() + raise ResizeFailedException(e) from e return (before, get_size(partdev)) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_grub_dpkg.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_grub_dpkg.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_grub_dpkg.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_grub_dpkg.py 2021-05-11 16:34:29.000000000 +0000 @@ -1,8 +1,9 @@ -# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2009-2010, 2020 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger +# Author: Matthew Ruffell # # This file is part of cloud-init. See LICENSE file for license information. @@ -15,15 +16,15 @@ should work correctly by default without any user configuration. It can be enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config dict. The global config key ``grub-dpkg`` is an alias for ``grub_dpkg``. If no -installation device is specified this module will look for the first existing -device in: +installation device is specified this module will execute grub-probe to +determine which disk the /boot directory is associated with. - - ``/dev/sda`` - - ``/dev/vda`` - - ``/dev/xvda`` - - ``/dev/sda1`` - - ``/dev/vda1`` - - ``/dev/xvda1`` +The value which is placed into the debconf database is in the format which the +grub postinstall script expects. Normally, this is a /dev/disk/by-id/ value, +but we do fallback to the plain disk name if a by-id name is not present. + +If this module is executed inside a container, then the debconf database is +seeded with empty values, and install_devices_empty is set to true. **Internal name:** ``cc_grub_dpkg`` @@ -42,11 +43,68 @@ import os +from cloudinit import subp from cloudinit import util +from cloudinit.subp import ProcessExecutionError distros = ['ubuntu', 'debian'] +def fetch_idevs(log): + """ + Fetches the /dev/disk/by-id device grub is installed to. + Falls back to plain disk name if no by-id entry is present. + """ + disk = "" + devices = [] + + try: + # get the root disk where the /boot directory resides. + disk = subp.subp(['grub-probe', '-t', 'disk', '/boot'], + capture=True)[0].strip() + except ProcessExecutionError as e: + # grub-common may not be installed, especially on containers + # FileNotFoundError is a nested exception of ProcessExecutionError + if isinstance(e.reason, FileNotFoundError): + log.debug("'grub-probe' not found in $PATH") + # disks from the container host are present in /proc and /sys + # which is where grub-probe determines where /boot is. + # it then checks for existence in /dev, which fails as host disks + # are not exposed to the container. + elif "failed to get canonical path" in e.stderr: + log.debug("grub-probe 'failed to get canonical path'") + else: + # something bad has happened, continue to log the error + raise + except Exception: + util.logexc(log, "grub-probe failed to execute for grub-dpkg") + + if not disk or not os.path.exists(disk): + # If we failed to detect a disk, we can return early + return '' + + try: + # check if disk exists and use udevadm to fetch symlinks + devices = subp.subp( + ['udevadm', 'info', '--root', '--query=symlink', disk], + capture=True + )[0].strip().split() + except Exception: + util.logexc( + log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk + ) + + log.debug('considering these device symlinks: %s', ','.join(devices)) + # filter symlinks for /dev/disk/by-id entries + devices = [dev for dev in devices if 'disk/by-id' in dev] + log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices)) + # select first device if there is one, else fall back to plain name + idevs = sorted(devices)[0] if devices else disk + log.debug('selected %s', idevs) + + return idevs + + def handle(name, cfg, _cloud, log, _args): mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) @@ -62,22 +120,10 @@ idevs_empty = util.get_cfg_option_str( mycfg, "grub-pc/install_devices_empty", None) - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs is None: - idevs = "" - if idevs_empty is None: - idevs_empty = "true" - else: - if idevs_empty is None: - idevs_empty = "false" - if idevs is None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/xvda", - "/dev/sda1", "/dev/vda1", "/dev/xvda1"): - if os.path.exists(dev): - idevs = dev - break + if idevs is None: + idevs = fetch_idevs(log) + if idevs_empty is None: + idevs_empty = "false" if idevs else "true" # now idevs and idevs_empty are set to determined values # or, those set by user @@ -90,7 +136,7 @@ (idevs, idevs_empty)) try: - util.subp(['debconf-set-selections'], dconf_sel) + subp.subp(['debconf-set-selections'], dconf_sel) except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_keys_to_console.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_keys_to_console.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_keys_to_console.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_keys_to_console.py 2021-05-11 16:34:29.000000000 +0000 @@ -9,14 +9,17 @@ """ Keys to Console --------------- -**Summary:** control which SSH keys may be written to console +**Summary:** control which SSH host keys may be written to console -For security reasons it may be desirable not to write SSH fingerprints and keys -to the console. To avoid the fingerprint of types of SSH keys being written to -console the ``ssh_fp_console_blacklist`` config key can be used. By default all -types of keys will have their fingerprints written to console. To avoid keys -of a key type being written to console the ``ssh_key_console_blacklist`` config -key can be used. By default ``ssh-dss`` keys are not written to console. +For security reasons it may be desirable not to write SSH host keys and their +fingerprints to the console. To avoid either being written to the console the +``emit_keys_to_console`` config key under the main ``ssh`` config key can be +used. To avoid the fingerprint of types of SSH host keys being written to +console the ``ssh_fp_console_blacklist`` config key can be used. By default +all types of keys will have their fingerprints written to console. To avoid +host keys of a key type being written to console the +``ssh_key_console_blacklist`` config key can be used. By default ``ssh-dss`` +host keys are not written to console. **Internal name:** ``cc_keys_to_console`` @@ -26,6 +29,9 @@ **Config keys**:: + ssh: + emit_keys_to_console: false + ssh_fp_console_blacklist: ssh_key_console_blacklist: """ @@ -33,6 +39,7 @@ import os from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -50,6 +57,11 @@ def handle(name, cfg, cloud, log, _args): + if util.is_false(cfg.get("ssh", {}).get("emit_keys_to_console", True)): + log.debug(("Skipping module named %s, " + "logging of SSH host keys disabled"), name) + return + helper_path = _get_helper_tool_path(cloud.distro) if not os.path.exists(helper_path): log.warning(("Unable to activate module %s," @@ -64,7 +76,7 @@ try: cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)] - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) except Exception: diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_landscape.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_landscape.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_landscape.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_landscape.py 2021-05-11 16:34:29.000000000 +0000 @@ -61,6 +61,7 @@ from configobj import ConfigObj from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -116,7 +117,7 @@ log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE) util.write_file(LS_DEFAULT_FILE, "RUN=1\n") - util.subp(["service", "landscape-client", "restart"]) + subp.subp(["service", "landscape-client", "restart"]) def merge_together(objs): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_locale.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_locale.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_locale.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_locale.py 2021-05-11 16:34:29.000000000 +0000 @@ -6,27 +6,58 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Locale ------- -**Summary:** set system locale +"""Locale: set system locale""" -Configure the system locale and apply it system wide. By default use the locale -specified by the datasource. +from textwrap import dedent -**Internal name:** ``cc_locale`` - -**Module frequency:** per instance +from cloudinit import util +from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema +from cloudinit.settings import PER_INSTANCE -**Supported distros:** all -**Config keys**:: +frequency = PER_INSTANCE +distros = ['all'] +schema = { + 'id': 'cc_locale', + 'name': 'Locale', + 'title': 'Set system locale', + 'description': dedent( + """\ + Configure the system locale and apply it system wide. By default use + the locale specified by the datasource.""" + ), + 'distros': distros, + 'examples': [ + dedent("""\ + # Set the locale to ar_AE + locale: ar_AE + """), + dedent("""\ + # Set the locale to fr_CA in /etc/alternate_path/locale + locale: fr_CA + locale_configfile: /etc/alternate_path/locale + """), + ], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'locale': { + 'type': 'string', + 'description': ( + "The locale to set as the system's locale (e.g. ar_PS)" + ), + }, + 'locale_configfile': { + 'type': 'string', + 'description': ( + "The file in which to write the locale configuration (defaults" + " to the distro's default location)" + ), + }, + }, +} - locale: - locale_configfile: -""" - -from cloudinit import util +__doc__ = get_schema_doc(schema) # Supplement python help() def handle(name, cfg, cloud, log, args): @@ -40,6 +71,8 @@ name, locale) return + validate_cloudconfig_schema(cfg, schema) + log.debug("Setting locale to %s", locale) locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile") cloud.distro.apply_locale(locale, locale_cfgfile) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_lxd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_lxd.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_lxd.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_lxd.py 2021-05-11 16:34:29.000000000 +0000 @@ -48,6 +48,7 @@ """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import os @@ -85,16 +86,16 @@ # Install the needed packages packages = [] - if not util.which("lxd"): + if not subp.which("lxd"): packages.append('lxd') - if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): + if init_cfg.get("storage_backend") == "zfs" and not subp.which('zfs'): packages.append('zfsutils-linux') if len(packages): try: cloud.distro.install_packages(packages) - except util.ProcessExecutionError as exc: + except subp.ProcessExecutionError as exc: log.warning("failed to install packages %s: %s", packages, exc) return @@ -104,20 +105,20 @@ 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - util.subp(['lxd', 'waitready', '--timeout=300']) + subp.subp(['lxd', 'waitready', '--timeout=300']) cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): cmd.extend(["--%s=%s" % (k.replace('_', '-'), str(init_cfg[k]))]) - util.subp(cmd) + subp.subp(cmd) # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" if bridge_cfg: net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) if os.path.exists("/etc/default/lxd-bridge") \ - and util.which(dconf_comm): + and subp.which(dconf_comm): # Bridge configured through packaging debconf = bridge_to_debconf(bridge_cfg) @@ -127,7 +128,7 @@ log.debug("Setting lxd debconf via " + dconf_comm) data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" - util.subp(['debconf-communicate'], data) + subp.subp(['debconf-communicate'], data) except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) @@ -137,7 +138,7 @@ # Run reconfigure log.debug("Running dpkg-reconfigure for lxd") - util.subp(['dpkg-reconfigure', 'lxd', + subp.subp(['dpkg-reconfigure', 'lxd', '--frontend=noninteractive']) else: # Built-in LXD bridge support @@ -264,7 +265,7 @@ env = {'LC_ALL': 'C', 'HOME': os.environ.get('HOME', '/root'), 'USER': os.environ.get('USER', 'root')} - util.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) + subp.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) def maybe_cleanup_default(net_name, did_init, create, attach, @@ -282,21 +283,25 @@ fail_assume_enoent = "failed. Assuming it did not exist." succeeded = "succeeded." if create: - msg = "Deletion of lxd network '%s' %s" + msg = "Detach of lxd network '%s' from profile '%s' %s" try: - _lxc(["network", "delete", net_name]) - LOG.debug(msg, net_name, succeeded) - except util.ProcessExecutionError as e: + _lxc(["network", "detach-profile", net_name, profile]) + LOG.debug(msg, net_name, profile, succeeded) + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e - LOG.debug(msg, net_name, fail_assume_enoent) + LOG.debug(msg, net_name, profile, fail_assume_enoent) + else: + msg = "Deletion of lxd network '%s' %s" + _lxc(["network", "delete", net_name]) + LOG.debug(msg, net_name, succeeded) if attach: msg = "Removal of device '%s' from profile '%s' %s" try: _lxc(["profile", "device", "remove", profile, nic_name]) LOG.debug(msg, nic_name, profile, succeeded) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e LOG.debug(msg, nic_name, profile, fail_assume_enoent) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_mcollective.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_mcollective.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_mcollective.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_mcollective.py 2021-05-11 16:34:29.000000000 +0000 @@ -56,6 +56,7 @@ from configobj import ConfigObj from cloudinit import log as logging +from cloudinit import subp from cloudinit import util PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem" @@ -140,6 +141,6 @@ configure(config=mcollective_cfg['conf']) # restart mcollective to handle updated config - util.subp(['service', 'mcollective', 'restart'], capture=False) + subp.subp(['service', 'mcollective', 'restart'], capture=False) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_mounts.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_mounts.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_mounts.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_mounts.py 2021-05-11 16:34:29.000000000 +0000 @@ -65,15 +65,19 @@ from string import whitespace import logging -import os.path +import os import re from cloudinit import type_utils +from cloudinit import subp from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) +# Name matches 'server:/path' +NETWORK_NAME_FILTER = r"^.+:.*" +NETWORK_NAME_RE = re.compile(NETWORK_NAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" MNT_COMMENT = "comment=cloudconfig" @@ -93,6 +97,13 @@ return False +def is_network_device(name): + # return true if this is a network device + if NETWORK_NAME_RE.match(name): + return True + return False + + def _get_nth_partition_for_device(device_path, partition_number): potential_suffixes = [str(partition_number), 'p%s' % (partition_number,), '-part%s' % (partition_number,)] @@ -122,6 +133,9 @@ devname = "ephemeral0" log.debug("Adjusted mount option from ephemeral to ephemeral0") + if is_network_device(startname): + return startname + device_path, partition_number = util.expand_dotted_devname(devname) if is_meta_device_name(device_path): @@ -223,46 +237,48 @@ return size -def create_swapfile(fname, size): +def create_swapfile(fname: str, size: str) -> None: """Size is in MiB.""" - errmsg = "Failed to create swapfile '%s' of size %dMB via %s: %s" + errmsg = "Failed to create swapfile '%s' of size %sMB via %s: %s" def create_swap(fname, size, method): LOG.debug("Creating swapfile in '%s' on fstype '%s' using '%s'", fname, fstype, method) if method == "fallocate": - cmd = ['fallocate', '-l', '%dM' % size, fname] + cmd = ['fallocate', '-l', '%sM' % size, fname] elif method == "dd": cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=1M', - 'count=%d' % size] + 'count=%s' % size] try: - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: - LOG.warning(errmsg, fname, size, method, e) + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: + LOG.info(errmsg, fname, size, method, e) util.del_file(fname) + raise swap_dir = os.path.dirname(fname) util.ensure_dir(swap_dir) fstype = util.get_mount_info(swap_dir)[1] - if fstype in ("xfs", "btrfs"): + if (fstype == "xfs" and + util.kernel_version() < (4, 18)) or fstype == "btrfs": create_swap(fname, size, "dd") else: try: create_swap(fname, size, "fallocate") - except util.ProcessExecutionError as e: - LOG.warning(errmsg, fname, size, "dd", e) - LOG.warning("Will attempt with dd.") + except subp.ProcessExecutionError: + LOG.info("fallocate swap creation failed, will attempt with dd") create_swap(fname, size, "dd") - util.chmod(fname, 0o600) + if os.path.exists(fname): + util.chmod(fname, 0o600) try: - util.subp(['mkswap', fname]) - except util.ProcessExecutionError: + subp.subp(['mkswap', fname]) + except subp.ProcessExecutionError: util.del_file(fname) raise @@ -274,7 +290,6 @@ maxsize: the maximum size """ swap_dir = os.path.dirname(fname) - mibsize = str(int(size / (2 ** 20))) if str(size).lower() == "auto": try: memsize = util.read_meminfo()['total'] @@ -286,6 +301,7 @@ size = suggested_swapsize(fsys=swap_dir, maxsize=maxsize, memsize=memsize) + mibsize = str(int(size / (2 ** 20))) if not size: LOG.debug("Not creating swap: suggested size was 0") return @@ -365,17 +381,18 @@ fstab_devs = {} fstab_removed = [] - for line in util.load_file(FSTAB_PATH).splitlines(): - if MNT_COMMENT in line: - fstab_removed.append(line) - continue + if os.path.exists(FSTAB_PATH): + for line in util.load_file(FSTAB_PATH).splitlines(): + if MNT_COMMENT in line: + fstab_removed.append(line) + continue - try: - toks = WS.split(line) - except Exception: - pass - fstab_devs[toks[0]] = line - fstab_lines.append(line) + try: + toks = WS.split(line) + except Exception: + pass + fstab_devs[toks[0]] = line + fstab_lines.append(line) for i in range(len(cfgmnt)): # skip something that wasn't a list @@ -525,9 +542,9 @@ for cmd in activate_cmds: fmt = "Activate mounts: %s:" + ' '.join(cmd) try: - util.subp(cmd) + subp.subp(cmd) log.debug(fmt, "PASS") - except util.ProcessExecutionError: + except subp.ProcessExecutionError: log.warning(fmt, "FAIL") util.logexc(log, fmt, "FAIL") diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ntp.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ntp.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ntp.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ntp.py 2021-05-11 16:34:29.000000000 +0000 @@ -14,6 +14,7 @@ from cloudinit import temp_utils from cloudinit import templater from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE @@ -23,7 +24,8 @@ frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' NR_POOL_SERVERS = 4 -distros = ['centos', 'debian', 'fedora', 'opensuse', 'rhel', 'sles', 'ubuntu'] +distros = ['almalinux', 'alpine', 'centos', 'debian', 'fedora', 'opensuse', + 'rhel', 'sles', 'ubuntu'] NTP_CLIENT_CONFIG = { 'chrony': { @@ -62,11 +64,30 @@ # This is Distro-specific configuration overrides of the base config DISTRO_CLIENT_CONFIG = { + 'alpine': { + 'chrony': { + 'confpath': '/etc/chrony/chrony.conf', + 'service_name': 'chronyd', + }, + 'ntp': { + 'confpath': '/etc/ntp.conf', + 'packages': [], + 'service_name': 'ntpd', + }, + }, 'debian': { 'chrony': { 'confpath': '/etc/chrony/chrony.conf', }, }, + 'rhel': { + 'ntp': { + 'service_name': 'ntpd', + }, + 'chrony': { + 'service_name': 'chronyd', + }, + }, 'opensuse': { 'chrony': { 'service_name': 'chronyd', @@ -113,11 +134,11 @@ Handle ntp configuration. If ntp is not installed on the system and ntp configuration is specified, ntp will be installed. If there is a default ntp config file in the image or one is present in the - distro's ntp package, it will be copied to ``/etc/ntp.conf.dist`` - before any changes are made. A list of ntp pools and ntp servers can - be provided under the ``ntp`` config key. If no ntp ``servers`` or - ``pools`` are provided, 4 pools will be used in the format - ``{0-3}.{distro}.pool.ntp.org``."""), + distro's ntp package, it will be copied to a file with ``.dist`` + appended to the filename before any changes are made. A list of ntp + pools and ntp servers can be provided under the ``ntp`` config key. + If no ntp ``servers`` or ``pools`` are provided, 4 pools will be used + in the format ``{0-3}.{distro}.pool.ntp.org``."""), 'distros': distros, 'examples': [ dedent("""\ @@ -169,8 +190,11 @@ 'uniqueItems': True, 'description': dedent("""\ List of ntp pools. If both pools and servers are - empty, 4 default pool servers will be provided of - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided of + the format ``{0-3}.{distro}.pool.ntp.org``. NOTE: + for Alpine Linux when using the Busybox NTP client + this setting will be ignored due to the limited + functionality of Busybox's ntpd.""") }, 'servers': { 'type': 'array', @@ -181,46 +205,46 @@ 'uniqueItems': True, 'description': dedent("""\ List of ntp servers. If both pools and servers are - empty, 4 default pool servers will be provided with - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided with + the format ``{0-3}.{distro}.pool.ntp.org``.""") }, 'ntp_client': { 'type': 'string', 'default': 'auto', 'description': dedent("""\ Name of an NTP client to use to configure system NTP. - When unprovided or 'auto' the default client preferred - by the distribution will be used. The following - built-in client names can be used to override existing - configuration defaults: chrony, ntp, ntpdate, - systemd-timesyncd."""), + When unprovided or 'auto' the default client preferred + by the distribution will be used. The following + built-in client names can be used to override existing + configuration defaults: chrony, ntp, ntpdate, + systemd-timesyncd."""), }, 'enabled': { 'type': 'boolean', 'default': True, 'description': dedent("""\ Attempt to enable ntp clients if set to True. If set - to False, ntp client will not be configured or - installed"""), + to False, ntp client will not be configured or + installed"""), }, 'config': { 'description': dedent("""\ Configuration settings or overrides for the - ``ntp_client`` specified."""), + ``ntp_client`` specified."""), 'type': ['object'], 'properties': { 'confpath': { 'type': 'string', 'description': dedent("""\ The path to where the ``ntp_client`` - configuration is written."""), + configuration is written."""), }, 'check_exe': { 'type': 'string', 'description': dedent("""\ The executable name for the ``ntp_client``. - For example, ntp service ``check_exe`` is - 'ntpd' because it runs the ntpd binary."""), + For example, ntp service ``check_exe`` is + 'ntpd' because it runs the ntpd binary."""), }, 'packages': { 'type': 'array', @@ -230,22 +254,22 @@ 'uniqueItems': True, 'description': dedent("""\ List of packages needed to be installed for the - selected ``ntp_client``."""), + selected ``ntp_client``."""), }, 'service_name': { 'type': 'string', 'description': dedent("""\ The systemd or sysvinit service name used to - start and stop the ``ntp_client`` - service."""), + start and stop the ``ntp_client`` + service."""), }, 'template': { 'type': 'string', 'description': dedent("""\ Inline template allowing users to define their - own ``ntp_client`` configuration template. - The value must start with '## template:jinja' - to enable use of templating support. + own ``ntp_client`` configuration template. + The value must start with '## template:jinja' + to enable use of templating support. """), }, }, @@ -307,7 +331,7 @@ if distro_ntp_client == "auto": for client in distro.preferred_ntp_clients: cfg = distro_cfg.get(client) - if util.which(cfg.get('check_exe')): + if subp.which(cfg.get('check_exe')): LOG.debug('Selected NTP client "%s", already installed', client) clientcfg = cfg @@ -336,7 +360,7 @@ @param check_exe: string. The name of a binary that indicates the package the specified package is already installed. """ - if util.which(check_exe): + if subp.which(check_exe): return if packages is None: packages = ['ntp'] @@ -363,21 +387,30 @@ """ names = [] pool_distro = distro - # For legal reasons x.pool.sles.ntp.org does not exist, - # use the opensuse pool + if distro == 'sles': + # For legal reasons x.pool.sles.ntp.org does not exist, + # use the opensuse pool pool_distro = 'opensuse' + elif distro == 'alpine': + # Alpine-specific pool (i.e. x.alpine.pool.ntp.org) does not exist + # so use general x.pool.ntp.org instead. + pool_distro = '' + for x in range(0, NR_POOL_SERVERS): - name = "%d.%s.pool.ntp.org" % (x, pool_distro) - names.append(name) + names.append(".".join( + [n for n in [str(x)] + [pool_distro] + ['pool.ntp.org'] if n])) + return names -def write_ntp_config_template(distro_name, servers=None, pools=None, - path=None, template_fn=None, template=None): +def write_ntp_config_template(distro_name, service_name=None, servers=None, + pools=None, path=None, template_fn=None, + template=None): """Render a ntp client configuration for the specified client. @param distro_name: string. The distro class name. + @param service_name: string. The name of the NTP client service. @param servers: A list of strings specifying ntp servers. Defaults to empty list. @param pools: A list of strings specifying ntp pools. Defaults to empty @@ -396,7 +429,14 @@ if not pools: pools = [] - if len(servers) == 0 and len(pools) == 0: + if (len(servers) == 0 and distro_name == 'alpine' and + service_name == 'ntpd'): + # Alpine's Busybox ntpd only understands "servers" configuration + # and not "pool" configuration. + servers = generate_server_names(distro_name) + LOG.debug( + 'Adding distro default ntp servers: %s', ','.join(servers)) + elif len(servers) == 0 and len(pools) == 0: pools = generate_server_names(distro_name) LOG.debug( 'Adding distro default ntp pool servers: %s', ','.join(pools)) @@ -431,7 +471,7 @@ cmd = ['systemctl', 'reload-or-restart', service] else: cmd = ['service', service, 'restart'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def supplemental_schema_validation(ntp_config): @@ -531,6 +571,8 @@ raise RuntimeError(msg) write_ntp_config_template(cloud.distro.name, + service_name=ntp_client_config.get( + 'service_name'), servers=ntp_cfg.get('servers', []), pools=ntp_cfg.get('pools', []), path=ntp_client_config.get('confpath'), @@ -543,7 +585,7 @@ try: reload_ntp(ntp_client_config['service_name'], systemd=cloud.distro.uses_systemd()) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.exception("Failed to reload/start ntp service: %s", e) raise diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_package_update_upgrade_install.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_package_update_upgrade_install.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_package_update_upgrade_install.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_package_update_upgrade_install.py 2021-05-11 16:34:29.000000000 +0000 @@ -43,6 +43,7 @@ import time from cloudinit import log as logging +from cloudinit import subp from cloudinit import util REBOOT_FILE = "/var/run/reboot-required" @@ -57,7 +58,7 @@ def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) + subp.subp(REBOOT_CMD) start = time.time() wait_time = initial_sleep for _i in range(0, wait_attempts): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_phone_home.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_phone_home.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_phone_home.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_phone_home.py 2021-05-11 16:34:29.000000000 +0000 @@ -19,6 +19,7 @@ - ``pub_key_dsa`` - ``pub_key_rsa`` - ``pub_key_ecdsa`` + - ``pub_key_ed25519`` - ``instance_id`` - ``hostname`` - ``fdqn`` @@ -52,6 +53,7 @@ 'pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', + 'pub_key_ed25519', 'instance_id', 'hostname', 'fqdn' @@ -105,6 +107,7 @@ 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', + 'pub_key_ed25519': '/etc/ssh/ssh_host_ed25519_key.pub', } for (n, path) in pubkeys.items(): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_power_state_change.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_power_state_change.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_power_state_change.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_power_state_change.py 2021-05-11 16:34:29.000000000 +0000 @@ -22,9 +22,8 @@ used. Therefore, if a 5 minute delay and a 120 second shutdown are specified, the maximum amount of time between cloud-init starting and the system shutting down is 7 minutes, and the minimum amount of time is 5 minutes. The ``delay`` -key must have an argument in a form that the ``shutdown`` utility recognizes. -The most common format is the form ``+5`` for 5 minutes. See ``man shutdown`` -for more options. +key must have an argument in either the form ``'+5'`` for 5 minutes or ``now`` +for immediate shutdown. Optionally, a command can be run to determine whether or not the system should shut down. The command to be run should be specified in the @@ -33,6 +32,10 @@ ``condition`` key is omitted or the command specified by the ``condition`` key returns 0. +.. note:: + With Alpine Linux any message value specified is ignored as Alpine's halt, + poweroff, and reboot commands do not support broadcasting a message. + **Internal name:** ``cc_power_state_change`` **Module frequency:** per instance @@ -56,6 +59,7 @@ import time from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -71,7 +75,7 @@ # PID COMM ARGS # 1 init /bin/init -- if util.is_FreeBSD(): - (output, _err) = util.subp(['procstat', '-c', str(pid)]) + (output, _err) = subp.subp(['procstat', '-c', str(pid)]) line = output.splitlines()[1] m = re.search(r'\d+ (\w|\.|-)+\s+(/\w.+)', line) return m.group(2) @@ -111,9 +115,9 @@ return False -def handle(_name, cfg, _cloud, log, _args): +def handle(_name, cfg, cloud, log, _args): try: - (args, timeout, condition) = load_power_state(cfg) + (args, timeout, condition) = load_power_state(cfg, cloud.distro) if args is None: log.debug("no power_state provided. doing nothing") return @@ -140,7 +144,7 @@ condition, execmd, [args, devnull_fp]) -def load_power_state(cfg): +def load_power_state(cfg, distro): # returns a tuple of shutdown_command, timeout # shutdown_command is None if no config found pstate = cfg.get('power_state') @@ -151,35 +155,23 @@ if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") - opt_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} - + modes_ok = ['halt', 'poweroff', 'reboot'] mode = pstate.get("mode") - if mode not in opt_map: + if mode not in distro.shutdown_options_map: raise TypeError( "power_state[mode] required, must be one of: %s. found: '%s'." % - (','.join(opt_map.keys()), mode)) - - delay = pstate.get("delay", "now") - # convert integer 30 or string '30' to '+30' - try: - delay = "+%s" % int(delay) - except ValueError: - pass - - if delay != "now" and not re.match(r"\+[0-9]+", delay): - raise TypeError( - "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay) + (','.join(modes_ok), mode)) - args = ["shutdown", opt_map[mode], delay] - if pstate.get("message"): - args.append(pstate.get("message")) + args = distro.shutdown_command(mode=mode, + delay=pstate.get("delay", "now"), + message=pstate.get("message")) try: timeout = float(pstate.get('timeout', 30.0)) - except ValueError: - raise ValueError("failed to convert timeout '%s' to float." % - pstate['timeout']) + except ValueError as e: + raise ValueError( + "failed to convert timeout '%s' to float." % pstate['timeout'] + ) from e condition = pstate.get("condition", True) if not isinstance(condition, (str, list, bool)): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_puppet.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_puppet.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_puppet.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_puppet.py 2021-05-11 16:34:29.000000000 +0000 @@ -83,6 +83,7 @@ from io import StringIO from cloudinit import helpers +from cloudinit import subp from cloudinit import util PUPPET_CONF_PATH = '/etc/puppet/puppet.conf' @@ -105,14 +106,14 @@ def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): - util.subp(['sed', '-i', + subp.subp(['sed', '-i', '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + subp.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) elif os.path.exists('/sbin/chkconfig'): - util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) + subp.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: log.warning(("Sorry we do not know how to enable" " puppet services on this system")) @@ -159,9 +160,9 @@ cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = '\n'.join(cleaned_lines) # Move to puppet_config.read_file when dropping py2.7 - puppet_config.readfp( # pylint: disable=W1505 + puppet_config.read_file( StringIO(cleaned_contents), - filename=p_constants.conf_path) + source=p_constants.conf_path) for (cfg_name, cfg) in puppet_cfg['conf'].items(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place @@ -203,6 +204,6 @@ _autostart_puppet(log) # Start puppetd - util.subp(['service', 'puppet', 'start'], capture=False) + subp.subp(['service', 'puppet', 'start'], capture=False) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_refresh_rmc_and_interface.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_refresh_rmc_and_interface.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_refresh_rmc_and_interface.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_refresh_rmc_and_interface.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,159 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + +""" +Refresh IPv6 interface and RMC +------------------------------ +**Summary:** Ensure Network Manager is not managing IPv6 interface + +This module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components +that together provide a comprehensive clustering environment(RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource +Monitoring and Control (RMC) subsystem. RMC is a generalized framework used +for managing, monitoring, and manipulating resources. RMC runs as a daemon +process on individual machines and needs creation of unique node id and +restarts during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- Refreshing RMC +- Disabling NetworkManager from handling IPv6 interface, as IPv6 interface + is used for communication between RMC daemon and PowerVM hypervisor. + +**Internal name:** ``cc_refresh_rmc_and_interface`` + +**Module frequency:** per always + +**Supported distros:** RHEL + +""" + +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util +from cloudinit import subp +from cloudinit import netinfo + +import errno + +frequency = PER_ALWAYS + +LOG = logging.getLogger(__name__) +# Ensure that /opt/rsct/bin has been added to standard PATH of the +# distro. The symlink to rmcctrl is /usr/sbin/rsct/bin/rmcctrl . +RMCCTRL = 'rmcctrl' + + +def handle(name, _cfg, _cloud, _log, _args): + if not subp.which(RMCCTRL): + LOG.debug("No '%s' in path, disabled", RMCCTRL) + return + + LOG.debug( + 'Making the IPv6 up explicitly. ' + 'Ensuring IPv6 interface is not being handled by NetworkManager ' + 'and it is restarted to re-establish the communication with ' + 'the hypervisor') + + ifaces = find_ipv6_ifaces() + + # Setting NM_CONTROLLED=no for IPv6 interface + # making it down and up + + if len(ifaces) == 0: + LOG.debug("Did not find any interfaces with ipv6 addresses.") + else: + for iface in ifaces: + refresh_ipv6(iface) + disable_ipv6(sysconfig_path(iface)) + restart_network_manager() + + +def find_ipv6_ifaces(): + info = netinfo.netdev_info() + ifaces = [] + for iface, data in info.items(): + if iface == "lo": + LOG.debug('Skipping localhost interface') + if len(data.get("ipv4", [])) != 0: + # skip this interface, as it has ipv4 addrs + continue + ifaces.append(iface) + return ifaces + + +def refresh_ipv6(interface): + # IPv6 interface is explicitly brought up, subsequent to which the + # RMC services are restarted to re-establish the communication with + # the hypervisor. + subp.subp(['ip', 'link', 'set', interface, 'down']) + subp.subp(['ip', 'link', 'set', interface, 'up']) + + +def sysconfig_path(iface): + return '/etc/sysconfig/network-scripts/ifcfg-' + iface + + +def restart_network_manager(): + subp.subp(['systemctl', 'restart', 'NetworkManager']) + + +def disable_ipv6(iface_file): + # Ensuring that the communication b/w the hypervisor and VM is not + # interrupted due to NetworkManager. For this purpose, as part of + # this function, the NM_CONTROLLED is explicitly set to No for IPV6 + # interface and NetworkManager is restarted. + try: + contents = util.load_file(iface_file) + except IOError as e: + if e.errno == errno.ENOENT: + LOG.debug("IPv6 interface file %s does not exist\n", + iface_file) + else: + raise e + + if 'IPV6INIT' not in contents: + LOG.debug("Interface file %s did not have IPV6INIT", iface_file) + return + + LOG.debug("Editing interface file %s ", iface_file) + + # Dropping any NM_CONTROLLED or IPV6 lines from IPv6 interface file. + lines = contents.splitlines() + lines = [line for line in lines if not search(line)] + lines.append("NM_CONTROLLED=no") + + with open(iface_file, "w") as fp: + fp.write("\n".join(lines) + "\n") + + +def search(contents): + # Search for any NM_CONTROLLED or IPV6 lines in IPv6 interface file. + return( + contents.startswith("IPV6ADDR") or + contents.startswith("IPADDR6") or + contents.startswith("IPV6INIT") or + contents.startswith("NM_CONTROLLED")) + + +def refresh_rmc(): + # To make a healthy connection between RMC daemon and hypervisor we + # refresh RMC. With refreshing RMC we are ensuring that making IPv6 + # down and up shouldn't impact communication between RMC daemon and + # hypervisor. + # -z : stop Resource Monitoring & Control subsystem and all resource + # managers, but the command does not return control to the user + # until the subsystem and all resource managers are stopped. + # -s : start Resource Monitoring & Control subsystem. + try: + subp.subp([RMCCTRL, '-z']) + subp.subp([RMCCTRL, '-s']) + except Exception: + util.logexc(LOG, 'Failed to refresh the RMC subsystem.') + raise diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_reset_rmc.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_reset_rmc.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_reset_rmc.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_reset_rmc.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,143 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + + +""" +Reset RMC +------------ +**Summary:** reset rsct node id + +Reset RMC module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components, +that together provide a comprehensive clustering environment (RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource monitoring +and control (RMC) subsystem. RMC is a generalized framework used for managing, +monitoring, and manipulating resources. RMC runs as a daemon process on +individual machines and needs creation of unique node id and restarts +during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- creation of the unique RSCT node id to every instance/virtual machine + and ensure once set, it isn't changed subsequently by cloud-init. + In order to do so, it restarts RSCT service. + +Prerequisite of using this module is to install RSCT packages. + +**Internal name:** ``cc_reset_rmc`` + +**Module frequency:** per instance + +**Supported distros:** rhel, sles and ubuntu + +""" +import os + +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import util +from cloudinit import subp + +frequency = PER_INSTANCE + +# RMCCTRL is expected to be in system PATH (/opt/rsct/bin) +# The symlink for RMCCTRL and RECFGCT are +# /usr/sbin/rsct/bin/rmcctrl and +# /usr/sbin/rsct/install/bin/recfgct respectively. +RSCT_PATH = '/opt/rsct/install/bin' +RMCCTRL = 'rmcctrl' +RECFGCT = 'recfgct' + +LOG = logging.getLogger(__name__) + +NODE_ID_FILE = '/etc/ct_node_id' + + +def handle(name, _cfg, cloud, _log, _args): + # Ensuring node id has to be generated only once during first boot + if cloud.datasource.platform_type == 'none': + LOG.debug('Skipping creation of new ct_node_id node') + return + + if not os.path.isdir(RSCT_PATH): + LOG.debug("module disabled, RSCT_PATH not present") + return + + orig_path = os.environ.get('PATH') + try: + add_path(orig_path) + reset_rmc() + finally: + if orig_path: + os.environ['PATH'] = orig_path + else: + del os.environ['PATH'] + + +def reconfigure_rsct_subsystems(): + # Reconfigure the RSCT subsystems, which includes removing all RSCT data + # under the /var/ct directory, generating a new node ID, and making it + # appear as if the RSCT components were just installed + try: + out = subp.subp([RECFGCT])[0] + LOG.debug(out.strip()) + return out + except subp.ProcessExecutionError: + util.logexc(LOG, 'Failed to reconfigure the RSCT subsystems.') + raise + + +def get_node_id(): + try: + fp = util.load_file(NODE_ID_FILE) + node_id = fp.split('\n')[0] + return node_id + except Exception: + util.logexc(LOG, 'Failed to get node ID from file %s.' % NODE_ID_FILE) + raise + + +def add_path(orig_path): + # Adding the RSCT_PATH to env standard path + # So thet cloud init automatically find and + # run RECFGCT to create new node_id. + suff = ":" + orig_path if orig_path else "" + os.environ['PATH'] = RSCT_PATH + suff + return os.environ['PATH'] + + +def rmcctrl(): + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + try: + return subp.subp([RMCCTRL, '-z']) + except Exception: + util.logexc(LOG, 'Failed to stop the RMC subsystem.') + raise + + +def reset_rmc(): + LOG.debug('Attempting to reset RMC.') + + node_id_before = get_node_id() + LOG.debug('Node ID at beginning of module: %s', node_id_before) + + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + rmcctrl() + reconfigure_rsct_subsystems() + + node_id_after = get_node_id() + LOG.debug('Node ID at end of module: %s', node_id_after) + + # Check if new node ID is generated or not + # by comparing old and new node ID + if node_id_after == node_id_before: + msg = 'New node ID did not get generated.' + LOG.error(msg) + raise Exception(msg) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_resizefs.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_resizefs.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_resizefs.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_resizefs.py 2021-05-11 16:34:29.000000000 +0000 @@ -9,16 +9,14 @@ """Resizefs: cloud-config module which resizes the filesystem""" import errno -import getopt import os -import re -import shlex import stat from textwrap import dedent from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util NOBLOCK = "noblock" @@ -87,58 +85,23 @@ return ('zpool', 'online', '-e', mount_point, devpth) -def _get_dumpfs_output(mount_point): - return util.subp(['dumpfs', '-m', mount_point])[0] - - -def _get_gpart_output(part): - return util.subp(['gpart', 'show', part])[0] - - def _can_skip_resize_ufs(mount_point, devpth): - # extract the current fs sector size - """ - # dumpfs -m / - # newfs command for / (/dev/label/rootfs) - newfs -L rootf -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 -f 4096 -g 16384 - -h 64 -i 8192 -j -k 6408 -m 8 -o time -s 58719232 /dev/label/rootf - """ - cur_fs_sz = None - frag_sz = None - dumpfs_res = _get_dumpfs_output(mount_point) - for line in dumpfs_res.splitlines(): - if not line.startswith('#'): - newfs_cmd = shlex.split(line) - opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:L:' - optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) - for o, a in optlist: - if o == "-s": - cur_fs_sz = int(a) - if o == "-f": - frag_sz = int(a) - # check the current partition size - """ - # gpart show /dev/da0 -=> 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """ - expect_sz = None - m = re.search('^(/dev/.+)p([0-9])$', devpth) - gpart_res = _get_gpart_output(m.group(1)) - for line in gpart_res.splitlines(): - if re.search(r"freebsd-ufs", line): - fields = line.split() - expect_sz = int(fields[1]) - # Normalize the gpart sector size, - # because the size is not exactly the same as fs size. - normal_expect_sz = (expect_sz - expect_sz % (frag_sz / 512)) - if normal_expect_sz == cur_fs_sz: - return True - else: - return False + # possible errors cases on the code-path to growfs -N following: + # https://github.com/freebsd/freebsd/blob/HEAD/sbin/growfs/growfs.c + # This is the "good" error: + skip_start = "growfs: requested size" + skip_contain = "is not larger than the current filesystem size" + # growfs exits with 1 for almost all cases up to this one. + # This means we can't just use rcs=[0, 1] as subp parameter: + try: + subp.subp(['growfs', '-N', devpth]) + except subp.ProcessExecutionError as e: + if e.stderr.startswith(skip_start) and skip_contain in e.stderr: + # This FS is already at the desired size + return True + else: + raise e + return False # Do not use a dictionary as these commands should be able to be used @@ -306,8 +269,8 @@ def do_resize(resize_cmd, log): try: - util.subp(resize_cmd) - except util.ProcessExecutionError: + subp.subp(resize_cmd) + except subp.ProcessExecutionError: util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise # TODO(harlowja): Should we add a fsck check after this to make diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_resolv_conf.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_resolv_conf.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_resolv_conf.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_resolv_conf.py 2021-05-11 16:34:29.000000000 +0000 @@ -14,12 +14,12 @@ This module is intended to manage resolv.conf in environments where early configuration of resolv.conf is necessary for further bootstrapping and/or where configuration management such as puppet or chef own dns configuration. -As Debian/Ubuntu will, by default, utilize resovlconf, and similarly RedHat +As Debian/Ubuntu will, by default, utilize resolvconf, and similarly Red Hat will use sysconfig, this module is likely to be of little use unless those are configured correctly. .. note:: - For RedHat with sysconfig, be sure to set PEERDNS=no for all DHCP + For Red Hat with sysconfig, be sure to set PEERDNS=no for all DHCP enabled NICs. .. note:: @@ -30,7 +30,7 @@ **Module frequency:** per instance -**Supported distros:** fedora, rhel, sles +**Supported distros:** alpine, fedora, rhel, sles **Config keys**:: @@ -55,7 +55,7 @@ frequency = PER_INSTANCE -distros = ['fedora', 'opensuse', 'rhel', 'sles'] +distros = ['alpine', 'fedora', 'opensuse', 'rhel', 'sles'] def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_rh_subscription.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_rh_subscription.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_rh_subscription.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_rh_subscription.py 2021-05-11 16:34:29.000000000 +0000 @@ -5,15 +5,15 @@ # This file is part of cloud-init. See LICENSE file for license information. """ -RedHat Subscription -------------------- +Red Hat Subscription +-------------------- **Summary:** register red hat enterprise linux based system -Register a RedHat system either by username and password *or* activation and +Register a Red Hat system either by username and password *or* activation and org. Following a sucessful registration, you can auto-attach subscriptions, set the service level, add subscriptions based on pool id, enable/disable yum repositories based on repo id, and alter the rhsm_baseurl and server-hostname -in ``/etc/rhsm/rhs.conf``. For more details, see the ``Register RedHat +in ``/etc/rhsm/rhs.conf``. For more details, see the ``Register Red Hat Subscription`` example config. **Internal name:** ``cc_rh_subscription`` @@ -39,6 +39,7 @@ """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -173,7 +174,7 @@ try: _sub_man_cli(cmd) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: return False return True @@ -200,7 +201,7 @@ try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -223,7 +224,7 @@ # Attempting to register the system only try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -246,7 +247,7 @@ try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout.rstrip() != '': for line in e.stdout.split("\n"): if line != '': @@ -264,7 +265,7 @@ cmd = ['attach', '--auto'] try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Auto-attach failed with: {0}".format(e)) return False for line in return_out.split("\n"): @@ -341,7 +342,7 @@ "system: %s", (", ".join(pool_list)) .replace('--pool=', '')) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to attach pool {0} " "due to {1}".format(pool, e)) return False @@ -414,7 +415,7 @@ try: _sub_man_cli(cmd) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to alter repos due to {0}".format(e)) return False @@ -432,11 +433,11 @@ def _sub_man_cli(cmd, logstring_val=False): ''' - Uses the prefered cloud-init subprocess def of util.subp + Uses the prefered cloud-init subprocess def of subp.subp and runs subscription-manager. Breaking this to a separate function for later use in mocking and unittests ''' - return util.subp(['subscription-manager'] + cmd, + return subp.subp(['subscription-manager'] + cmd, logstring=logstring_val) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_rightscale_userdata.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_rightscale_userdata.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_rightscale_userdata.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_rightscale_userdata.py 2021-05-11 16:34:29.000000000 +0000 @@ -44,7 +44,7 @@ # - read the blob of data from raw user data, and parse it as key/value # - for each key that is found, download the content to # the local instance/scripts directory and set them executable. -# - the files in that directory will be run by the user-scripts module +# - the files in that directory will be run by the scripts-user module # Therefore, this must run before that. # # diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_rsyslog.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_rsyslog.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_rsyslog.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_rsyslog.py 2021-05-11 16:34:29.000000000 +0000 @@ -182,6 +182,7 @@ import re from cloudinit import log as logging +from cloudinit import subp from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" @@ -215,7 +216,7 @@ cmd = ['service', service, 'restart'] else: cmd = command - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def load_config(cfg): @@ -346,8 +347,10 @@ if self.port: try: int(self.port) - except ValueError: - raise ValueError("port '%s' is not an integer" % self.port) + except ValueError as e: + raise ValueError( + "port '%s' is not an integer" % self.port + ) from e if not self.addr: raise ValueError("address is required") @@ -429,7 +432,7 @@ restarted = reload_syslog( command=mycfg[KEYNAME_RELOAD], systemd=cloud.distro.uses_systemd()), - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: restarted = False log.warning("Failed to reload syslog", e) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_salt_minion.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_salt_minion.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_salt_minion.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_salt_minion.py 2021-05-11 16:34:29.000000000 +0000 @@ -45,7 +45,7 @@ import os -from cloudinit import safeyaml, util +from cloudinit import safeyaml, subp, util from cloudinit.distros import rhel_util @@ -130,6 +130,6 @@ # restart salt-minion. 'service' will start even if not started. if it # was started, it needs to be restarted for config change. - util.subp(['service', const.srv_name, 'restart'], capture=False) + subp.subp(['service', const.srv_name, 'restart'], capture=False) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_per_boot.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_per_boot.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_per_boot.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_per_boot.py 2021-05-11 16:34:29.000000000 +0000 @@ -24,7 +24,7 @@ import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_ALWAYS @@ -38,7 +38,7 @@ # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_per_instance.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_per_instance.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_per_instance.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_per_instance.py 2021-05-11 16:34:29.000000000 +0000 @@ -27,7 +27,7 @@ import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_INSTANCE @@ -41,7 +41,7 @@ # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_per_once.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_per_once.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_per_once.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_per_once.py 2021-05-11 16:34:29.000000000 +0000 @@ -25,7 +25,7 @@ import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_ONCE @@ -39,7 +39,7 @@ # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_user.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_user.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_user.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_user.py 2021-05-11 16:34:29.000000000 +0000 @@ -27,7 +27,7 @@ import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_INSTANCE @@ -42,7 +42,7 @@ # go here... runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_vendor.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_vendor.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_scripts_vendor.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_scripts_vendor.py 2021-05-11 16:34:29.000000000 +0000 @@ -28,6 +28,7 @@ import os +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -46,7 +47,7 @@ prefix = util.get_cfg_by_path(cfg, ('vendor_data', 'prefix'), []) try: - util.runparts(runparts_path, exe_prefix=prefix) + subp.runparts(runparts_path, exe_prefix=prefix) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_seed_random.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_seed_random.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_seed_random.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_seed_random.py 2021-05-11 16:34:29.000000000 +0000 @@ -24,15 +24,19 @@ optionally be specified in encoded form, with the encoding specified in ``encoding``. +If the cloud provides its own random seed data, it will be appended to ``data`` +before it is written to ``file``. + .. note:: when using a multiline value for ``data`` or specifying binary data, be sure to follow yaml syntax and use the ``|`` and ``!binary`` yaml format specifiers when appropriate -Instead of specifying a data string, a command can be run to generate/collect -the data to be written. The command should be specified as a list of args in -the ``command`` key. If a command is specified that cannot be run, no error -will be reported unless ``command_required`` is set to true. +If the ``command`` key is specified, the given command will be executed. This +will happen after ``file`` has been populated. That command's environment will +contain the value of the ``file`` key as ``RANDOM_SEED_FILE``. If a command is +specified that cannot be run, no error will be reported unless +``command_required`` is set to true. For example, to use ``pollinate`` to gather data from a remote entropy server and write it to ``/dev/urandom``, the following could be @@ -65,6 +69,7 @@ from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -92,14 +97,14 @@ return cmd = command[0] - if not util.which(cmd): + if not subp.which(cmd): if required: raise ValueError( "command '{cmd}' not found but required=true".format(cmd=cmd)) else: LOG.debug("command '%s' not found for seed_command", cmd) return - util.subp(command, env=env, capture=False) + subp.subp(command, env=env, capture=False) def handle(name, cfg, cloud, log, _args): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_set_hostname.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_set_hostname.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_set_hostname.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_set_hostname.py 2021-05-11 16:34:29.000000000 +0000 @@ -18,8 +18,11 @@ ``fqdn`` key. Alternatively, a hostname can be specified using the ``hostname`` key, and the fqdn of the cloud wil be used. If a fqdn specified with the ``hostname`` key, it will be handled properly, although it is better to use -the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set, ``fqdn`` -will be used. +the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set, +it is distro dependent whether ``hostname`` or ``fqdn`` is used, +unless the ``prefer_fqdn_over_hostname`` option is true and fqdn is set +it will force the use of FQDN in all distros, and if false then it will +force the hostname use. This module will run in the init-local stage before networking is configured if the hostname is set by metadata or user data on the local system. @@ -38,6 +41,7 @@ **Config keys**:: preserve_hostname: + prefer_fqdn_over_hostname: fqdn: hostname: """ @@ -55,7 +59,6 @@ This may happen if we attempt to set the hostname early in cloud-init's init-local timeframe as certain services may not be running yet. """ - pass def handle(name, cfg, cloud, log, _args): @@ -63,6 +66,14 @@ log.debug(("Configuration option 'preserve_hostname' is set," " not setting the hostname in module %s"), name) return + + # Set prefer_fqdn_over_hostname value in distro + hostname_fqdn = util.get_cfg_option_bool(cfg, + "prefer_fqdn_over_hostname", + None) + if hostname_fqdn is not None: + cloud.distro.set_option('prefer_fqdn_over_hostname', hostname_fqdn) + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) # Check for previous successful invocation of set-hostname @@ -86,7 +97,7 @@ except Exception as e: msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) util.logexc(log, msg) - raise SetHostnameError("%s: %s" % (msg, e)) + raise SetHostnameError("%s: %s" % (msg, e)) from e write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn}) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_set_passwords.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_set_passwords.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_set_passwords.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_set_passwords.py 2021-05-11 16:34:29.000000000 +0000 @@ -78,11 +78,11 @@ """ import re -import sys from cloudinit.distros import ug_util from cloudinit import log as logging from cloudinit.ssh_util import update_ssh_config +from cloudinit import subp from cloudinit import util from string import ascii_letters, digits @@ -128,7 +128,7 @@ cmd = list(service_cmd) + ["restart", service_name] else: cmd = list(service_cmd) + [service_name, "restart"] - util.subp(cmd) + subp.subp(cmd) LOG.debug("Restarted the SSH daemon.") @@ -213,7 +213,9 @@ if len(randlist): blurb = ("Set the following 'random' passwords\n", '\n'.join(randlist)) - sys.stderr.write("%s\n%s\n" % blurb) + util.multi_log( + "%s\n%s\n" % blurb, stderr=False, fallback_to_stdout=False + ) if expire: expired_users = [] @@ -241,12 +243,12 @@ def chpasswd(distro, plist_in, hashed=False): - if util.is_FreeBSD(): + if util.is_BSD(): for pentry in plist_in.splitlines(): u, p = pentry.split(":") distro.set_passwd(u, p, hashed=hashed) else: cmd = ['chpasswd'] + (['-e'] if hashed else []) - util.subp(cmd, plist_in) + subp.subp(cmd, plist_in) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_snap.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_snap.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_snap.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_snap.py 2021-05-11 16:34:29.000000000 +0000 @@ -12,6 +12,7 @@ get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE from cloudinit.subp import prepend_base_command +from cloudinit import subp from cloudinit import util @@ -61,9 +62,9 @@ snap: assertions: 00: | - signed_assertion_blob_here + signed_assertion_blob_here 02: | - signed_assertion_blob_here + signed_assertion_blob_here commands: 00: snap create-user --sudoer --known @mydomain.com 01: snap install canonical-livepatch @@ -85,6 +86,21 @@ 01: ['snap', 'install', 'vlc'] 02: snap install vlc 03: 'snap install vlc' + """), dedent("""\ + # You can use a list of commands + snap: + commands: + - ['install', 'vlc'] + - ['snap', 'install', 'vlc'] + - snap install vlc + - 'snap install vlc' + """), dedent("""\ + # You can use a list of assertions + snap: + assertions: + - signed_assertion_blob_here + - | + signed_assertion_blob_here """)], 'frequency': PER_INSTANCE, 'type': 'object', @@ -98,7 +114,8 @@ 'additionalItems': False, # Reject items non-string 'minItems': 1, 'minProperties': 1, - 'uniqueItems': True + 'uniqueItems': True, + 'additionalProperties': {'type': 'string'}, }, 'commands': { 'type': ['object', 'array'], # Array of strings or dict @@ -110,6 +127,12 @@ 'additionalItems': False, # Reject non-string & non-list 'minItems': 1, 'minProperties': 1, + 'additionalProperties': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'array', 'items': {'type': 'string'}}, + ], + }, }, 'squashfuse_in_container': { 'type': 'boolean' @@ -122,10 +145,6 @@ } } -# TODO schema for 'assertions' and 'commands' are too permissive at the moment. -# Once python-jsonschema supports schema draft 6 add support for arbitrary -# object keys with 'patternProperties' constraint to validate string values. - __doc__ = get_schema_doc(schema) # Supplement python help() SNAP_CMD = "snap" @@ -157,7 +176,7 @@ LOG.debug('Snap acking: %s', asrt.split('\n')[0:2]) util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) - util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + subp.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) def run_commands(commands): @@ -186,8 +205,8 @@ for command in fixed_snap_commands: shell = isinstance(command, str) try: - util.subp(command, shell=shell, status_cb=sys.stderr.write) - except util.ProcessExecutionError as e: + subp.subp(command, shell=shell, status_cb=sys.stderr.write) + except subp.ProcessExecutionError as e: cmd_failures.append(str(e)) if cmd_failures: msg = 'Failures running snap commands:\n{cmd_failures}'.format( diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_spacewalk.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_spacewalk.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_spacewalk.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_spacewalk.py 2021-05-11 16:34:29.000000000 +0000 @@ -27,7 +27,7 @@ activation_key: """ -from cloudinit import util +from cloudinit import subp distros = ['redhat', 'fedora'] @@ -41,9 +41,9 @@ # assume we aren't registered; which is sorta ghetto... already_registered = False try: - util.subp(['rhn-profile-sync', '--verbose'], capture=False) + subp.subp(['rhn-profile-sync', '--verbose'], capture=False) already_registered = True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise return already_registered @@ -65,7 +65,7 @@ cmd.extend(['--sslCACert', str(ca_cert_path)]) if activation_key: cmd.extend(['--activationkey', str(activation_key)]) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def handle(name, cfg, cloud, log, _args): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ssh_authkey_fingerprints.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ssh_authkey_fingerprints.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ssh_authkey_fingerprints.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ssh_authkey_fingerprints.py 2021-05-11 16:34:29.000000000 +0000 @@ -13,7 +13,7 @@ default, but can be disabled using ``no_ssh_fingerprints``. The hash type for the keys can be specified, but defaults to ``sha256``. -**Internal name:** `` cc_ssh_authkey_fingerprints`` +**Internal name:** ``cc_ssh_authkey_fingerprints`` **Module frequency:** per instance @@ -59,8 +59,8 @@ def _is_printable_key(entry): if any([entry.keytype, entry.base64, entry.comment, entry.options]): - if (entry.keytype and - entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): + if (entry.keytype and entry.keytype.lower().strip() + in ssh_util.VALID_KEY_TYPES): return True return False diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ssh_import_id.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ssh_import_id.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ssh_import_id.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ssh_import_id.py 2021-05-11 16:34:29.000000000 +0000 @@ -31,6 +31,7 @@ """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util import pwd @@ -101,8 +102,8 @@ log.debug("Importing SSH ids for user %s.", user) try: - util.subp(cmd, capture=False) - except util.ProcessExecutionError as exc: + subp.subp(cmd, capture=False) + except subp.ProcessExecutionError as exc: util.logexc(log, "Failed to run command to import %s SSH ids", user) raise exc diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ssh.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ssh.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ssh.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ssh.py 2021-05-11 16:34:29.000000000 +0000 @@ -35,6 +35,42 @@ no-port-forwarding,no-agent-forwarding,no-X11-forwarding +Supported public key types for the ``ssh_authorized_keys`` are: + + - dsa + - rsa + - ecdsa + - ed25519 + - ecdsa-sha2-nistp256-cert-v01@openssh.com + - ecdsa-sha2-nistp256 + - ecdsa-sha2-nistp384-cert-v01@openssh.com + - ecdsa-sha2-nistp384 + - ecdsa-sha2-nistp521-cert-v01@openssh.com + - ecdsa-sha2-nistp521 + - sk-ecdsa-sha2-nistp256-cert-v01@openssh.com + - sk-ecdsa-sha2-nistp256@openssh.com + - sk-ssh-ed25519-cert-v01@openssh.com + - sk-ssh-ed25519@openssh.com + - ssh-dss-cert-v01@openssh.com + - ssh-dss + - ssh-ed25519-cert-v01@openssh.com + - ssh-ed25519 + - ssh-rsa-cert-v01@openssh.com + - ssh-rsa + - ssh-xmss-cert-v01@openssh.com + - ssh-xmss@openssh.com + +.. note:: + this list has been filtered out from the supported keytypes of + `OpenSSH`_ source, where the sigonly keys are removed. Please see + ``ssh_util`` for more information. + + ``dsa``, ``rsa``, ``ecdsa`` and ``ed25519`` are added for legacy, + as they are valid public keys in some old distros. They can possibly + be removed in the future when support for the older distros are dropped + +.. _OpenSSH: https://github.com/openssh/openssh-portable/blob/master/sshkey.c + Host Keys ^^^^^^^^^ @@ -47,8 +83,9 @@ Host keys can be added using the ``ssh_keys`` configuration key. The argument to this config key should be a dictionary entries for the public and private keys of each desired key type. Entries in the ``ssh_keys`` config dict should -have keys in the format ``_private`` and ``_public``, -e.g. ``rsa_private: `` and ``rsa_public: ``. See below for supported +have keys in the format ``_private``, ``_public``, and, +optionally, ``_certificate``, e.g. ``rsa_private: ``, +``rsa_public: ``, and ``rsa_certificate: ``. See below for supported key types. Not all key types have to be specified, ones left unspecified will not be used. If this config option is used, then no keys will be generated. @@ -58,7 +95,8 @@ secure .. note:: - to specify multiline private host keys, use yaml multiline syntax + to specify multiline private host keys and certificates, use yaml + multiline syntax If no host keys are specified using ``ssh_keys``, then keys will be generated using ``ssh-keygen``. By default one public/private pair of each supported @@ -92,12 +130,17 @@ ... -----END RSA PRIVATE KEY----- rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ... + rsa_certificate: | + ssh-rsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt ... dsa_private: | -----BEGIN DSA PRIVATE KEY----- MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qco ... -----END DSA PRIVATE KEY----- dsa_public: ssh-dsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ... + dsa_certificate: | + ssh-dsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt ... + ssh_genkeytypes: disable_root: disable_root_opts: @@ -116,6 +159,7 @@ from cloudinit.distros import ug_util from cloudinit import ssh_util +from cloudinit import subp from cloudinit import util @@ -132,6 +176,8 @@ CONFIG_KEY_TO_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) CONFIG_KEY_TO_FILE.update( {"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) + CONFIG_KEY_TO_FILE.update( + {"%s_certificate" % k: (KEY_FILE_TPL % k + "-cert.pub", 0o600)}) PRIV_TO_PUB["%s_private" % k] = "%s_public" % k KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' @@ -149,12 +195,18 @@ util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them + # if there are keys and/or certificates in cloud-config, use them for (key, val) in cfg["ssh_keys"].items(): - if key in CONFIG_KEY_TO_FILE: - tgt_fn = CONFIG_KEY_TO_FILE[key][0] - tgt_perms = CONFIG_KEY_TO_FILE[key][1] - util.write_file(tgt_fn, val, tgt_perms) + # skip entry if unrecognized + if key not in CONFIG_KEY_TO_FILE: + continue + tgt_fn = CONFIG_KEY_TO_FILE[key][0] + tgt_perms = CONFIG_KEY_TO_FILE[key][1] + util.write_file(tgt_fn, val, tgt_perms) + # set server to present the most recently identified certificate + if '_certificate' in key: + cert_config = {'HostCertificate': tgt_fn} + ssh_util.update_ssh_config(cert_config) for (priv, pub) in PRIV_TO_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: @@ -164,7 +216,7 @@ try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except Exception: util.logexc(log, "Failed generated a key for %s from %s", @@ -186,9 +238,9 @@ # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - out, err = util.subp(cmd, capture=True, env=lang_c) + out, err = subp.subp(cmd, capture=True, env=lang_c) sys.stdout.write(util.decode_binary(out)) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() if (e.exit_code == 1 and err.lower().startswith("unknown key")): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ubuntu_advantage.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ubuntu_advantage.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ubuntu_advantage.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ubuntu_advantage.py 2021-05-11 16:34:29.000000000 +0000 @@ -8,6 +8,7 @@ get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util @@ -109,18 +110,18 @@ attach_cmd = ['ua', 'attach', token] LOG.debug('Attaching to Ubuntu Advantage. %s', ' '.join(attach_cmd)) try: - util.subp(attach_cmd) - except util.ProcessExecutionError as e: + subp.subp(attach_cmd) + except subp.ProcessExecutionError as e: msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) - raise RuntimeError(msg) + raise RuntimeError(msg) from e enable_errors = [] for service in enable: try: cmd = ['ua', 'enable', service] - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: enable_errors.append((service, e)) if enable_errors: for service, error in enable_errors: @@ -135,7 +136,7 @@ def maybe_install_ua_tools(cloud): """Install ubuntu-advantage-tools if not present.""" - if util.which('ua'): + if subp.which('ua'): return try: cloud.distro.update_package_sources() diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ubuntu_drivers.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ubuntu_drivers.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_ubuntu_drivers.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_ubuntu_drivers.py 2021-05-11 16:34:29.000000000 +0000 @@ -9,6 +9,7 @@ get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import util @@ -108,7 +109,7 @@ LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc) return - if not util.which('ubuntu-drivers'): + if not subp.which('ubuntu-drivers'): LOG.debug("'ubuntu-drivers' command not available. " "Installing ubuntu-drivers-common") pkg_install_func(['ubuntu-drivers-common']) @@ -131,7 +132,7 @@ debconf_script, util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT), mode=0o755) - util.subp([debconf_script, debconf_file]) + subp.subp([debconf_script, debconf_file]) except Exception as e: util.logexc( LOG, "Failed to register NVIDIA debconf template: %s", str(e)) @@ -141,8 +142,8 @@ util.del_dir(tdir) try: - util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) - except util.ProcessExecutionError as exc: + subp.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) + except subp.ProcessExecutionError as exc: if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr: LOG.warning('the available version of ubuntu-drivers is' ' too old to perform requested driver installation') diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_update_hostname.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_update_hostname.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_update_hostname.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_update_hostname.py 2021-05-11 16:34:29.000000000 +0000 @@ -27,6 +27,7 @@ **Config keys**:: preserve_hostname: + prefer_fqdn_over_hostname: fqdn: hostname: """ @@ -45,6 +46,13 @@ " not updating the hostname in module %s"), name) return + # Set prefer_fqdn_over_hostname value in distro + hostname_fqdn = util.get_cfg_option_bool(cfg, + "prefer_fqdn_over_hostname", + None) + if hostname_fqdn is not None: + cloud.distro.set_option('prefer_fqdn_over_hostname', hostname_fqdn) + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) try: prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname") diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_users_groups.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_users_groups.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_users_groups.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_users_groups.py 2021-05-11 16:34:29.000000000 +0000 @@ -26,13 +26,14 @@ config keys for an entry in ``users`` are as follows: - ``name``: The user's login name - - ``expiredate``: Optional. Date on which the user's login will be + - ``expiredate``: Optional. Date on which the user's account will be disabled. Default: none - ``gecos``: Optional. Comment about the user, usually a comma-separated string of real name and contact information. Default: none - ``groups``: Optional. Additional groups to add the user to. Default: none - ``homedir``: Optional. Home dir for user. Default is ``/home/`` - - ``inactive``: Optional. Mark user inactive. Default: false + - ``inactive``: Optional. Number of days after a password expires until + the account is permanently disabled. Default: none - ``lock_passwd``: Optional. Disable password login. Default: true - ``no_create_home``: Optional. Do not create home directory. Default: false @@ -78,6 +79,12 @@ If specifying a sudo rule for a user, ensure that the syntax for the rule is valid, as it is not checked by cloud-init. +.. note:: + Most of these configuration options will not be honored if the user + already exists. The following options are the exceptions; they are applied + to already-existing users: ``plain_text_passwd``, ``hashed_passwd``, + ``lock_passwd``, ``sudo``, ``ssh_authorized_keys``, ``ssh_redirect_user``. + **Internal name:** ``cc_users_groups`` **Module frequency:** per instance @@ -96,11 +103,11 @@ - name: sudo: false - name: - expiredate: + expiredate: '' gecos: groups: homedir: - inactive: + inactive: '' lock_passwd: no_create_home: no_log_init: diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_write_files.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_write_files.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_write_files.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_write_files.py 2021-05-11 16:34:29.000000000 +0000 @@ -4,72 +4,162 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Write Files ------------ -**Summary:** write arbitrary files +"""Write Files: write arbitrary files""" -Write out arbitrary content to files, optionally setting permissions. Content -can be specified in plain text or binary. Data encoded with either base64 or -binary gzip data can be specified and will be decoded before being written. +import base64 +import os +from textwrap import dedent -.. note:: - if multiline data is provided, care should be taken to ensure that it - follows yaml formatting standards. to specify binary data, use the yaml - option ``!!binary`` +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import util -.. note:: - Do not write files under /tmp during boot because of a race with - systemd-tmpfiles-clean that can cause temp files to get cleaned during - the early boot process. Use /run/somedir instead to avoid race LP:1707222. -**Internal name:** ``cc_write_files`` +frequency = PER_INSTANCE -**Module frequency:** per instance +DEFAULT_OWNER = "root:root" +DEFAULT_PERMS = 0o644 +UNKNOWN_ENC = 'text/plain' -**Supported distros:** all +LOG = logging.getLogger(__name__) -**Config keys**:: +distros = ['all'] - write_files: +# The schema definition for each cloud-config module is a strict contract for +# describing supported configuration parameters for each cloud-config section. +# It allows cloud-config to validate and alert users to invalid or ignored +# configuration options before actually attempting to deploy with said +# configuration. + +supported_encoding_types = [ + 'gz', 'gzip', 'gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64', 'b64', + 'base64'] + +schema = { + 'id': 'cc_write_files', + 'name': 'Write Files', + 'title': 'write arbitrary files', + 'description': dedent("""\ + Write out arbitrary content to files, optionally setting permissions. + Parent folders in the path are created if absent. + Content can be specified in plain text or binary. Data encoded with + either base64 or binary gzip data can be specified and will be decoded + before being written. For empty file creation, content can be omitted. + + .. note:: + if multiline data is provided, care should be taken to ensure that it + follows yaml formatting standards. to specify binary data, use the yaml + option ``!!binary`` + + .. note:: + Do not write files under /tmp during boot because of a race with + systemd-tmpfiles-clean that can cause temp files to get cleaned during + the early boot process. Use /run/somedir instead to avoid race + LP:1707222."""), + 'distros': distros, + 'examples': [ + dedent("""\ + # Write out base64 encoded content to /etc/sysconfig/selinux + write_files: - encoding: b64 content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4... owner: root:root path: /etc/sysconfig/selinux permissions: '0644' - - content: | - # My new /etc/sysconfig/samba file - - SMDBOPTIONS="-D" - path: /etc/sysconfig/samba - - content: !!binary | - f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAA - AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAA - AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAAB - ... - path: /bin/arch - permissions: '0555' + """), + dedent("""\ + # Appending content to an existing file + write_files: - content: | 15 * * * * root ship_logs path: /etc/crontab append: true -""" + """), + dedent("""\ + # Provide gziped binary content + write_files: + - encoding: gzip + content: !!binary | + H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= + path: /usr/bin/hello + permissions: '0755' + """), + dedent("""\ + # Create an empty file on the system + write_files: + - path: /root/CLOUD_INIT_WAS_HERE + """)], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'write_files': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'path': { + 'type': 'string', + 'description': dedent("""\ + Path of the file to which ``content`` is decoded + and written + """), + }, + 'content': { + 'type': 'string', + 'default': '', + 'description': dedent("""\ + Optional content to write to the provided ``path``. + When content is present and encoding is not '%s', + decode the content prior to writing. Default: + **''** + """ % UNKNOWN_ENC), + }, + 'owner': { + 'type': 'string', + 'default': DEFAULT_OWNER, + 'description': dedent("""\ + Optional owner:group to chown on the file. Default: + **{owner}** + """.format(owner=DEFAULT_OWNER)), + }, + 'permissions': { + 'type': 'string', + 'default': oct(DEFAULT_PERMS).replace('o', ''), + 'description': dedent("""\ + Optional file permissions to set on ``path`` + represented as an octal string '0###'. Default: + **'{perms}'** + """.format(perms=oct(DEFAULT_PERMS).replace('o', ''))), + }, + 'encoding': { + 'type': 'string', + 'default': UNKNOWN_ENC, + 'enum': supported_encoding_types, + 'description': dedent("""\ + Optional encoding type of the content. Default is + **text/plain** and no content decoding is + performed. Supported encoding types are: + %s.""" % ", ".join(supported_encoding_types)), + }, + 'append': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + Whether to append ``content`` to existing file if + ``path`` exists. Default: **false**. + """), + }, + }, + 'required': ['path'], + 'additionalProperties': False + }, + } + } +} -import base64 -import os - -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - - -frequency = PER_INSTANCE - -DEFAULT_OWNER = "root:root" -DEFAULT_PERMS = 0o644 -UNKNOWN_ENC = 'text/plain' - -LOG = logging.getLogger(__name__) +__doc__ = get_schema_doc(schema) # Supplement python help() def handle(name, cfg, _cloud, log, _args): @@ -78,6 +168,7 @@ log.debug(("Skipping module named %s," " no/empty 'write_files' key in configuration"), name) return + validate_cloudconfig_schema(cfg, schema) write_files(name, files) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/cc_yum_add_repo.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_yum_add_repo.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/cc_yum_add_repo.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/cc_yum_add_repo.py 2021-05-11 16:34:29.000000000 +0000 @@ -18,7 +18,7 @@ **Module frequency:** per always -**Supported distros:** fedora, rhel +**Supported distros:** almalinux, centos, fedora, rhel **Config keys**:: @@ -36,7 +36,7 @@ from cloudinit import util -distros = ['fedora', 'rhel'] +distros = ['almalinux', 'centos', 'fedora', 'rhel'] def _canonicalize_id(repo_id): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/schema.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/schema.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/schema.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/schema.py 2021-05-11 16:34:29.000000000 +0000 @@ -1,8 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """schema.py: Set of module functions for processing cloud-config schema.""" -from __future__ import print_function - +from cloudinit.cmd.devel import read_cfg_paths from cloudinit import importer from cloudinit.util import find_modules, load_file @@ -36,6 +35,8 @@ {examples} """ SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}' +SCHEMA_LIST_ITEM_TMPL = ( + '{prefix}Each item in **{prop_name}** list supports the following keys:') SCHEMA_EXAMPLES_HEADER = '\n**Examples**::\n\n' SCHEMA_EXAMPLES_SPACER_TEMPLATE = '\n # --- Example{0} ---' @@ -58,6 +59,19 @@ super(SchemaValidationError, self).__init__(message) +def is_schema_byte_string(checker, instance): + """TYPE_CHECKER override allowing bytes for string type + + For jsonschema v. 3.0.0+ + """ + try: + from jsonschema import Draft4Validator + except ImportError: + return False + return (Draft4Validator.TYPE_CHECKER.is_type(instance, "string") or + isinstance(instance, (bytes,))) + + def validate_cloudconfig_schema(config, schema, strict=False): """Validate provided config meets the schema definition. @@ -73,11 +87,31 @@ """ try: from jsonschema import Draft4Validator, FormatChecker + from jsonschema.validators import create, extend except ImportError: logging.debug( 'Ignoring schema validation. python-jsonschema is not present') return - validator = Draft4Validator(schema, format_checker=FormatChecker()) + + # Allow for bytes to be presented as an acceptable valid value for string + # type jsonschema attributes in cloud-init's schema. + # This allows #cloud-config to provide valid yaml "content: !!binary | ..." + if hasattr(Draft4Validator, 'TYPE_CHECKER'): # jsonschema 3.0+ + type_checker = Draft4Validator.TYPE_CHECKER.redefine( + 'string', is_schema_byte_string) + cloudinitValidator = extend(Draft4Validator, type_checker=type_checker) + else: # jsonschema 2.6 workaround + types = Draft4Validator.DEFAULT_TYPES + # Allow bytes as well as string (and disable a spurious + # unsupported-assignment-operation pylint warning which appears because + # this code path isn't written against the latest jsonschema). + types['string'] = (str, bytes) # pylint: disable=E1137 + cloudinitValidator = create( + meta_schema=Draft4Validator.META_SCHEMA, + validators=Draft4Validator.VALIDATORS, + version="draft4", + default_types=types) + validator = cloudinitValidator(schema, format_checker=FormatChecker()) errors = () for error in sorted(validator.iter_errors(config), key=lambda e: e.path): path = '.'.join([str(p) for p in error.path]) @@ -106,7 +140,6 @@ schemapaths = _schemapath_for_cloudconfig( cloudconfig, original_content) errors_by_line = defaultdict(list) - error_count = 1 error_footer = [] annotated_content = [] for path, msg in schema_errors: @@ -120,18 +153,17 @@ if col is not None: msg = 'Line {line} column {col}: {msg}'.format( line=line, col=col, msg=msg) - error_footer.append('# E{0}: {1}'.format(error_count, msg)) - error_count += 1 lines = original_content.decode().split('\n') - error_count = 1 - for line_number, line in enumerate(lines): - errors = errors_by_line[line_number + 1] + error_index = 1 + for line_number, line in enumerate(lines, 1): + errors = errors_by_line[line_number] if errors: - error_label = ','.join( - ['E{0}'.format(count + error_count) - for count in range(0, len(errors))]) - error_count += len(errors) - annotated_content.append(line + '\t\t# ' + error_label) + error_label = [] + for error in errors: + error_label.append('E{0}'.format(error_index)) + error_footer.append('# E{0}: {1}'.format(error_index, error)) + error_index += 1 + annotated_content.append(line + '\t\t# ' + ','.join(error_label)) else: annotated_content.append(line) annotated_content.append( @@ -142,7 +174,8 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): """Validate cloudconfig file adheres to a specific jsonschema. - @param config_path: Path to the yaml cloud-config file to parse. + @param config_path: Path to the yaml cloud-config file to parse, or None + to default to system userdata from Paths object. @param schema: Dict describing a valid jsonschema to validate against. @param annotate: Boolean set True to print original config file with error annotations on the offending lines. @@ -150,9 +183,24 @@ @raises SchemaValidationError containing any of schema_errors encountered. @raises RuntimeError when config_path does not exist. """ - if not os.path.exists(config_path): - raise RuntimeError('Configfile {0} does not exist'.format(config_path)) - content = load_file(config_path, decode=False) + if config_path is None: + # Use system's raw userdata path + if os.getuid() != 0: + raise RuntimeError( + "Unable to read system userdata as non-root user." + " Try using sudo" + ) + paths = read_cfg_paths() + user_data_file = paths.get_ipath_cur("userdata_raw") + content = load_file(user_data_file, decode=False) + else: + if not os.path.exists(config_path): + raise RuntimeError( + 'Configfile {0} does not exist'.format( + config_path + ) + ) + content = load_file(config_path, decode=False) if not content.startswith(CLOUD_CONFIG_HEADER): errors = ( ('format-l1.c1', 'File {0} needs to begin with "{1}"'.format( @@ -179,7 +227,7 @@ error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) - raise error + raise error from e try: validate_cloudconfig_schema( cloudconfig, schema, strict=True) @@ -213,20 +261,34 @@ previous_depth = -1 path_prefix = '' if line.startswith('- '): + # Process list items adding a list_index to the path prefix + previous_list_idx = '.%d' % (list_index - 1) + if path_prefix and path_prefix.endswith(previous_list_idx): + path_prefix = path_prefix[:-len(previous_list_idx)] key = str(list_index) - value = line[1:] + schema_line_numbers[key] = line_number + item_indent = len(re.match(RE_YAML_INDENT, line[1:]).groups()[0]) + item_indent += 1 # For the leading '-' character + previous_depth = indent_depth + indent_depth += item_indent + line = line[item_indent:] # Strip leading list item + whitespace list_index += 1 else: + # Process non-list lines setting value if present list_index = 0 key, value = line.split(':', 1) + if path_prefix: + # Append any existing path_prefix for a fully-pathed key + key = path_prefix + '.' + key while indent_depth <= previous_depth: if scopes: previous_depth, path_prefix = scopes.pop() + if list_index > 0 and indent_depth == previous_depth: + path_prefix = '.'.join(path_prefix.split('.')[:-1]) + break else: previous_depth = -1 path_prefix = '' - if path_prefix: - key = path_prefix + '.' + key scopes.append((indent_depth, key)) if value: value = value.strip() @@ -259,6 +321,28 @@ return property_type +def _parse_description(description, prefix): + """Parse description from the schema in a format that we can better + display in our docs. This parser does three things: + + - Guarantee that a paragraph will be in a single line + - Guarantee that each new paragraph will be aligned with + the first paragraph + - Proper align lists of items + + @param description: The original description in the schema. + @param prefix: The number of spaces used to align the current description + """ + list_paragraph = prefix * 3 + description = re.sub(r"(\S)\n(\S)", r"\1 \2", description) + description = re.sub( + r"\n\n", r"\n\n{}".format(prefix), description) + description = re.sub( + r"\n( +)-", r"\n{}-".format(list_paragraph), description) + + return description + + def _get_property_doc(schema, prefix=' '): """Return restructured text describing the supported schema properties.""" new_prefix = prefix + ' ' @@ -266,11 +350,23 @@ for prop_key, prop_config in schema.get('properties', {}).items(): # Define prop_name and dscription for SCHEMA_PROPERTY_TMPL description = prop_config.get('description', '') + properties.append(SCHEMA_PROPERTY_TMPL.format( prefix=prefix, prop_name=prop_key, type=_get_property_type(prop_config), - description=description.replace('\n', ''))) + description=_parse_description(description, prefix))) + items = prop_config.get('items') + if items: + if isinstance(items, list): + for item in items: + properties.append( + _get_property_doc(item, prefix=new_prefix)) + elif isinstance(items, dict) and items.get('properties'): + properties.append(SCHEMA_LIST_ITEM_TMPL.format( + prefix=new_prefix, prop_name=prop_key)) + new_prefix += ' ' + properties.append(_get_property_doc(items, prefix=new_prefix)) if 'properties' in prop_config: properties.append( _get_property_doc(prop_config, prefix=new_prefix)) @@ -346,8 +442,11 @@ description='Validate cloud-config files or document schema') parser.add_argument('-c', '--config-file', help='Path of the cloud-config yaml file to validate') - parser.add_argument('-d', '--doc', action="store_true", default=False, - help='Print schema documentation') + parser.add_argument('--system', action='store_true', default=False, + help='Validate the system cloud-config userdata') + parser.add_argument('-d', '--docs', nargs='+', + help=('Print schema module docs. Choices: all or' + ' space-delimited cc_names.')) parser.add_argument('--annotate', action="store_true", default=False, help='Annotate existing cloud-config file with errors') return parser @@ -355,11 +454,11 @@ def handle_schema_args(name, args): """Handle provided schema args and perform the appropriate actions.""" - exclusive_args = [args.config_file, args.doc] - if not any(exclusive_args) or all(exclusive_args): - error('Expected either --config-file argument or --doc') + exclusive_args = [args.config_file, args.docs, args.system] + if len([arg for arg in exclusive_args if arg]) != 1: + error('Expected one of --config-file, --system or --docs arguments') full_schema = get_schema() - if args.config_file: + if args.config_file or args.system: try: validate_cloudconfig_file( args.config_file, full_schema, args.annotate) @@ -369,10 +468,21 @@ except RuntimeError as e: error(str(e)) else: - print("Valid cloud-config file {0}".format(args.config_file)) - if args.doc: + if args.config_file is None: + cfg_name = "system userdata" + else: + cfg_name = args.config_file + print("Valid cloud-config:", cfg_name) + elif args.docs: + schema_ids = [subschema['id'] for subschema in full_schema['allOf']] + schema_ids += ['all'] + invalid_docs = set(args.docs).difference(set(schema_ids)) + if invalid_docs: + error('Invalid --docs value {0}. Must be one of: {1}'.format( + list(invalid_docs), ', '.join(schema_ids))) for subschema in full_schema['allOf']: - print(get_schema_doc(subschema)) + if 'all' in args.docs or subschema['id'] in args.docs: + print(get_schema_doc(subschema)) def main(): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_disable_ec2_metadata.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_disable_ec2_metadata.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_disable_ec2_metadata.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_disable_ec2_metadata.py 2021-05-11 16:34:29.000000000 +0000 @@ -15,10 +15,8 @@ class TestEC2MetadataRoute(CiTestCase): - with_logs = True - - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ifconfig(self, m_subp, m_which): """Set the route if ifconfig command is available""" m_which.side_effect = lambda x: x if x == 'ifconfig' else None @@ -27,8 +25,8 @@ ['route', 'add', '-host', '169.254.169.254', 'reject'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ip(self, m_subp, m_which): """Set the route if ip command is available""" m_which.side_effect = lambda x: x if x == 'ip' else None @@ -37,8 +35,8 @@ ['ip', 'route', 'add', 'prohibit', '169.254.169.254'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_no_tool(self, m_subp, m_which): """Log error when neither route nor ip commands are available""" m_which.return_value = None # Find neither ifconfig nor ip diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_final_message.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_final_message.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_final_message.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_final_message.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,46 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import logging +from unittest import mock + +import pytest + +from cloudinit.config.cc_final_message import handle + + +class TestHandle: + # TODO: Expand these tests to cover full functionality; currently they only + # cover the logic around how the boot-finished file is written (and not its + # contents). + + @pytest.mark.parametrize( + "instance_dir_exists,file_is_written,expected_log_substring", + [ + (True, True, None), + (False, False, "Failed to write boot finished file "), + ], + ) + def test_boot_finished_written( + self, + instance_dir_exists, + file_is_written, + expected_log_substring, + caplog, + tmpdir, + ): + instance_dir = tmpdir.join("var/lib/cloud/instance") + if instance_dir_exists: + instance_dir.ensure_dir() + boot_finished = instance_dir.join("boot-finished") + + m_cloud = mock.Mock( + paths=mock.Mock(boot_finished=boot_finished.strpath) + ) + + handle(None, {}, m_cloud, logging.getLogger(), []) + + # We should not change the status of the instance directory + assert instance_dir_exists == instance_dir.exists() + assert file_is_written == boot_finished.exists() + + if expected_log_substring: + assert expected_log_substring in caplog.text diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_grub_dpkg.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_grub_dpkg.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_grub_dpkg.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_grub_dpkg.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,176 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import pytest + +from unittest import mock +from logging import Logger +from cloudinit.subp import ProcessExecutionError +from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle + + +class TestFetchIdevs: + """Tests cc_grub_dpkg.fetch_idevs()""" + + # Note: udevadm info returns devices in a large single line string + @pytest.mark.parametrize( + "grub_output,path_exists,expected_log_call,udevadm_output" + ",expected_idevs", + [ + # Inside a container, grub not installed + ( + ProcessExecutionError(reason=FileNotFoundError()), + False, + mock.call("'grub-probe' not found in $PATH"), + '', + '', + ), + # Inside a container, grub installed + ( + ProcessExecutionError(stderr="failed to get canonical path"), + False, + mock.call("grub-probe 'failed to get canonical path'"), + '', + '', + ), + # KVM Instance + ( + ['/dev/vda'], + True, + None, + ( + '/dev/disk/by-path/pci-0000:00:00.0 ', + '/dev/disk/by-path/virtio-pci-0000:00:00.0 ' + ), + '/dev/vda', + ), + # Xen Instance + ( + ['/dev/xvda'], + True, + None, + '', + '/dev/xvda', + ), + # NVMe Hardware Instance + ( + ['/dev/nvme1n1'], + True, + None, + ( + '/dev/disk/by-id/nvme-Company_hash000 ', + '/dev/disk/by-id/nvme-nvme.000-000-000-000-000 ', + '/dev/disk/by-path/pci-0000:00:00.0-nvme-0 ' + ), + '/dev/disk/by-id/nvme-Company_hash000', + ), + # SCSI Hardware Instance + ( + ['/dev/sda'], + True, + None, + ( + '/dev/disk/by-id/company-user-1 ', + '/dev/disk/by-id/scsi-0Company_user-1 ', + '/dev/disk/by-path/pci-0000:00:00.0-scsi-0:0:0:0 ' + ), + '/dev/disk/by-id/company-user-1', + ), + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") + def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output, + path_exists, expected_log_call, udevadm_output, + expected_idevs): + """Tests outputs from grub-probe and udevadm info against grub-dpkg""" + m_subp.side_effect = [ + grub_output, + ["".join(udevadm_output)] + ] + m_exists.return_value = path_exists + log = mock.Mock(spec=Logger) + idevs = fetch_idevs(log) + assert expected_idevs == idevs + if expected_log_call is not None: + assert expected_log_call in log.debug.call_args_list + + +class TestHandle: + """Tests cc_grub_dpkg.handle()""" + + @pytest.mark.parametrize( + "cfg_idevs,cfg_idevs_empty,fetch_idevs_output,expected_log_output", + [ + ( + # No configuration + None, + None, + '/dev/disk/by-id/nvme-Company_hash000', + ( + "Setting grub debconf-set-selections with ", + "'/dev/disk/by-id/nvme-Company_hash000','false'" + ), + ), + ( + # idevs set, idevs_empty unset + '/dev/sda', + None, + '/dev/sda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/sda','false'" + ), + ), + ( + # idevs unset, idevs_empty set + None, + 'true', + '/dev/xvda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/xvda','true'" + ), + ), + ( + # idevs set, idevs_empty set + '/dev/vda', + 'false', + '/dev/disk/by-id/company-user-1', + ( + "Setting grub debconf-set-selections with ", + "'/dev/vda','false'" + ), + ), + ( + # idevs set, idevs_empty set + # Respect what the user defines, even if its logically wrong + '/dev/nvme0n1', + 'true', + '', + ( + "Setting grub debconf-set-selections with ", + "'/dev/nvme0n1','true'" + ), + ) + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") + def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs, + cfg_idevs, cfg_idevs_empty, fetch_idevs_output, + expected_log_output): + """Test setting of correct debconf database entries""" + m_get_cfg_str.side_effect = [ + cfg_idevs, + cfg_idevs_empty + ] + m_fetch_idevs.return_value = fetch_idevs_output + log = mock.Mock(spec=Logger) + handle(mock.Mock(), mock.Mock(), mock.Mock(), log, mock.Mock()) + log.debug.assert_called_with("".join(expected_log_output)) + + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_keys_to_console.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_keys_to_console.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_keys_to_console.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_keys_to_console.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,34 @@ +"""Tests for cc_keys_to_console.""" +from unittest import mock + +import pytest + +from cloudinit.config import cc_keys_to_console + + +class TestHandle: + """Tests for cloudinit.config.cc_keys_to_console.handle. + + TODO: These tests only cover the emit_keys_to_console config option, they + should be expanded to cover the full functionality. + """ + + @mock.patch("cloudinit.config.cc_keys_to_console.util.multi_log") + @mock.patch("cloudinit.config.cc_keys_to_console.os.path.exists") + @mock.patch("cloudinit.config.cc_keys_to_console.subp.subp") + @pytest.mark.parametrize("cfg,subp_called", [ + ({}, True), # Default to emitting keys + ({"ssh": {}}, True), # Default even if we have the parent key + ({"ssh": {"emit_keys_to_console": True}}, True), # Explicitly enabled + ({"ssh": {"emit_keys_to_console": False}}, False), # Disabled + ]) + def test_emit_keys_to_console_config( + self, m_subp, m_path_exists, _m_multi_log, cfg, subp_called + ): + # Ensure we always find the helper + m_path_exists.return_value = True + m_subp.return_value = ("", "") + + cc_keys_to_console.handle("name", cfg, mock.Mock(), mock.Mock(), ()) + + assert subp_called == (m_subp.call_count == 1) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_mounts.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_mounts.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_mounts.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_mounts.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,61 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + +import pytest + +from cloudinit.config.cc_mounts import create_swapfile +from cloudinit.subp import ProcessExecutionError + + +M_PATH = 'cloudinit.config.cc_mounts.' + + +class TestCreateSwapfile: + + @pytest.mark.parametrize('fstype', ('xfs', 'btrfs', 'ext4', 'other')) + @mock.patch(M_PATH + 'util.get_mount_info') + @mock.patch(M_PATH + 'subp.subp') + def test_happy_path(self, m_subp, m_get_mount_info, fstype, tmpdir): + swap_file = tmpdir.join("swap-file") + fname = str(swap_file) + + # Some of the calls to subp.subp should create the swap file; this + # roughly approximates that + m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') + + m_get_mount_info.return_value = (mock.ANY, fstype) + + create_swapfile(fname, '') + assert mock.call(['mkswap', fname]) in m_subp.call_args_list + + @mock.patch(M_PATH + "util.get_mount_info") + @mock.patch(M_PATH + "subp.subp") + def test_fallback_from_fallocate_to_dd( + self, m_subp, m_get_mount_info, caplog, tmpdir + ): + swap_file = tmpdir.join("swap-file") + fname = str(swap_file) + + def subp_side_effect(cmd, *args, **kwargs): + # Mock fallocate failing, to initiate fallback + if cmd[0] == "fallocate": + raise ProcessExecutionError() + + m_subp.side_effect = subp_side_effect + # Use ext4 so both fallocate and dd are valid swap creation methods + m_get_mount_info.return_value = (mock.ANY, "ext4") + + create_swapfile(fname, "") + + cmds = [args[0][0] for args, _kwargs in m_subp.call_args_list] + assert "fallocate" in cmds, "fallocate was not called" + assert "dd" in cmds, "fallocate failure did not fallback to dd" + + assert cmds.index("dd") > cmds.index( + "fallocate" + ), "dd ran before fallocate" + + assert mock.call(["mkswap", fname]) in m_subp.call_args_list + + msg = "fallocate swap creation failed, will attempt with dd" + assert msg in caplog.text diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_resolv_conf.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_resolv_conf.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_resolv_conf.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_resolv_conf.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,86 @@ +from unittest import mock + +import pytest + +from cloudinit.config.cc_resolv_conf import generate_resolv_conf + + +EXPECTED_HEADER = """\ +# Your system has been configured with 'manage-resolv-conf' set to true. +# As a result, cloud-init has written this file with configuration data +# that it has been provided. Cloud-init, by default, will write this file +# a single time (PER_ONCE). +#\n\n""" + + +class TestGenerateResolvConf: + @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") + def test_default_target_fname_is_etc_resolvconf(self, m_render_to_file): + generate_resolv_conf("templates/resolv.conf.tmpl", mock.MagicMock()) + + assert [ + mock.call(mock.ANY, "/etc/resolv.conf", mock.ANY) + ] == m_render_to_file.call_args_list + + @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") + def test_target_fname_is_used_if_passed(self, m_render_to_file): + generate_resolv_conf( + "templates/resolv.conf.tmpl", mock.MagicMock(), "/use/this/path" + ) + + assert [ + mock.call(mock.ANY, "/use/this/path", mock.ANY) + ] == m_render_to_file.call_args_list + + # Patch in templater so we can assert on the actual generated content + @mock.patch("cloudinit.templater.util.write_file") + # Parameterise with the value to be passed to generate_resolv_conf as the + # params parameter, and the expected line after the header as + # expected_extra_line. + @pytest.mark.parametrize( + "params,expected_extra_line", + [ + # No options + ({}, None), + # Just a true flag + ({"options": {"foo": True}}, "options foo"), + # Just a false flag + ({"options": {"foo": False}}, None), + # Just an option + ({"options": {"foo": "some_value"}}, "options foo:some_value"), + # A true flag and an option + ( + {"options": {"foo": "some_value", "bar": True}}, + "options bar foo:some_value", + ), + # Two options + ( + {"options": {"foo": "some_value", "bar": "other_value"}}, + "options bar:other_value foo:some_value", + ), + # Everything + ( + { + "options": { + "foo": "some_value", + "bar": "other_value", + "baz": False, + "spam": True, + } + }, + "options spam bar:other_value foo:some_value", + ), + ], + ) + def test_flags_and_options( + self, m_write_file, params, expected_extra_line + ): + generate_resolv_conf("templates/resolv.conf.tmpl", params) + + expected_content = EXPECTED_HEADER + if expected_extra_line is not None: + # If we have any extra lines, expect a trailing newline + expected_content += "\n".join([expected_extra_line, ""]) + assert [ + mock.call(mock.ANY, expected_content, mode=mock.ANY) + ] == m_write_file.call_args_list diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_set_passwords.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_set_passwords.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_set_passwords.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_set_passwords.py 2021-05-11 16:34:29.000000000 +0000 @@ -14,7 +14,7 @@ with_logs = True - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unknown_value_logs_warning(self, m_subp): setpass.handle_ssh_pwauth("floo") self.assertIn("Unrecognized value: ssh_pwauth=floo", @@ -22,7 +22,7 @@ m_subp.assert_not_called() @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_systemctl_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -31,7 +31,7 @@ m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_service_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -40,7 +40,7 @@ m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=False) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_not_restarted_if_not_updated(self, m_subp, m_update_ssh_config): """If config is not updated, then no system restart should be done.""" setpass.handle_ssh_pwauth(True) @@ -48,7 +48,7 @@ self.assertIn("No need to restart SSH", self.logs.getvalue()) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unchanged_does_nothing(self, m_subp, m_update_ssh_config): """If 'unchanged', then no updates to config and no restart.""" setpass.handle_ssh_pwauth( @@ -56,7 +56,7 @@ m_update_ssh_config.assert_not_called() m_subp.assert_not_called() - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_valid_change_values(self, m_subp): """If value is a valid changen value, then update should be called.""" upname = MODPATH + "update_ssh_config" @@ -74,10 +74,6 @@ with_logs = True - def setUp(self): - super(TestSetPasswordsHandle, self).setUp() - self.add_patch('cloudinit.config.cc_set_passwords.sys.stderr', 'm_err') - def test_handle_on_empty_config(self, *args): """handle logs that no password has changed when config is empty.""" cloud = self.tmp_cloud(distro='ubuntu') @@ -88,7 +84,7 @@ 'ssh_pwauth=None\n', self.logs.getvalue()) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_handle_on_chpasswd_list_parses_common_hashes(self, m_subp): """handle parses command password hashes.""" cloud = self.tmp_cloud(distro='ubuntu') @@ -98,7 +94,7 @@ 'ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52q' 'SDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1'] cfg = {'chpasswd': {'list': valid_hashed_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( @@ -112,12 +108,12 @@ '\n'.join(valid_hashed_pwds) + '\n')], m_subp.call_args_list) - @mock.patch(MODPATH + "util.is_FreeBSD") - @mock.patch(MODPATH + "util.subp") - def test_freebsd_calls_custom_pw_cmds_to_set_and_expire_passwords( - self, m_subp, m_is_freebsd): - """FreeBSD calls custom pw commands instead of chpasswd and passwd""" - m_is_freebsd.return_value = True + @mock.patch(MODPATH + "util.is_BSD") + @mock.patch(MODPATH + "subp.subp") + def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( + self, m_subp, m_is_bsd): + """BSD don't use chpasswd""" + m_is_bsd.return_value = True cloud = self.tmp_cloud(distro='freebsd') valid_pwds = ['ubuntu:passw0rd'] cfg = {'chpasswd': {'list': valid_pwds}} @@ -129,27 +125,51 @@ mock.call(['pw', 'usermod', 'ubuntu', '-p', '01-Jan-1970'])], m_subp.call_args_list) - @mock.patch(MODPATH + "util.is_FreeBSD") - @mock.patch(MODPATH + "util.subp") - def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp, - m_is_freebsd): + @mock.patch(MODPATH + "util.multi_log") + @mock.patch(MODPATH + "util.is_BSD") + @mock.patch(MODPATH + "subp.subp") + def test_handle_on_chpasswd_list_creates_random_passwords( + self, m_subp, m_is_bsd, m_multi_log + ): """handle parses command set random passwords.""" - m_is_freebsd.return_value = False + m_is_bsd.return_value = False cloud = self.tmp_cloud(distro='ubuntu') valid_random_pwds = [ 'root:R', 'ubuntu:RANDOM'] cfg = {'chpasswd': {'expire': 'false', 'list': valid_random_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( 'DEBUG: Handling input for chpasswd as list.', self.logs.getvalue()) - self.assertNotEqual( - [mock.call(['chpasswd'], - '\n'.join(valid_random_pwds) + '\n')], - m_subp.call_args_list) + + self.assertEqual(1, m_subp.call_count) + args, _kwargs = m_subp.call_args + self.assertEqual(["chpasswd"], args[0]) + + stdin = args[1] + user_pass = { + user: password + for user, password + in (line.split(":") for line in stdin.splitlines()) + } + + self.assertEqual(1, m_multi_log.call_count) + self.assertEqual( + mock.call(mock.ANY, stderr=False, fallback_to_stdout=False), + m_multi_log.call_args + ) + + self.assertEqual(set(["root", "ubuntu"]), set(user_pass.keys())) + written_lines = m_multi_log.call_args[0][0].splitlines() + for password in user_pass.values(): + for line in written_lines: + if password in line: + break + else: + self.fail("Password not emitted to console") # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_snap.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_snap.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_snap.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_snap.py 2021-05-11 16:34:29.000000000 +0000 @@ -92,7 +92,7 @@ super(TestAddAssertions, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_on_empty_list(self, m_subp): """When provided with an empty list, add_assertions does nothing.""" add_assertions([]) @@ -107,7 +107,7 @@ "assertion parameter was not a list or dict: I'm Not Valid", str(context_manager.exception)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_list(self, m_subp): """When provided with a list, add_assertions adds all assertions.""" self.assertEqual( @@ -130,7 +130,7 @@ self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_dict(self, m_subp): """When provided with a dict, add_assertions adds all assertions.""" self.assertEqual( @@ -168,7 +168,7 @@ super(TestRunCommands, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_run_commands_on_empty_list(self, m_subp): """When provided with an empty list, run_commands does nothing.""" run_commands([]) @@ -310,6 +310,52 @@ {'snap': {'commands': {'01': 'also valid'}}}, schema) self.assertEqual('', self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_values_are_invalid_type(self, _): + """Warnings when snap:commands values are invalid type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'commands': [123]}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': 123}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.commands.0: 123 is not valid under any of the given" + " schemas\n" + "WARNING: Invalid config:\n" + "snap.commands.01: 123 is not valid under any of the given" + " schemas\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_list_values_are_invalid_type(self, _): + """Warnings when snap:commands list values are wrong type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'commands': [["snap", "install", 123]]}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': ["snap", "install", 123]}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.commands.0: ['snap', 'install', 123] is not valid under any" + " of the given schemas\n", + "WARNING: Invalid config:\n" + "snap.commands.0: ['snap', 'install', 123] is not valid under any" + " of the given schemas\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_assertions_values_are_invalid_type(self, _): + """Warnings when snap:assertions values are invalid type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'assertions': [123]}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {'01': 123}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.assertions.0: 123 is not of type 'string'\n" + "WARNING: Invalid config:\n" + "snap.assertions.01: 123 is not of type 'string'\n", + self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.add_assertions') def test_warn_schema_assertions_is_not_list_or_dict(self, _): """Warn when snap:assertions config is not a list or dict.""" @@ -345,7 +391,7 @@ def test_duplicates_are_fine_array_array(self): """Duplicated commands array/array entries are allowed.""" self.assertSchemaValid( - {'commands': [["echo", "bye"], ["echo" "bye"]]}, + {'commands': [["echo", "bye"], ["echo", "bye"]]}, "command entries can be duplicate.") def test_duplicates_are_fine_array_string(self): @@ -431,7 +477,7 @@ self.assertEqual('HI\nMOM\n', util.load_file(outfile)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_handle_adds_assertions(self, m_subp): """Any configured snap assertions are provided to add_assertions.""" assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) @@ -447,7 +493,7 @@ self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') @skipUnlessJsonSchema() def test_handle_validates_schema(self, m_subp): """Any provided configuration is runs validate_cloudconfig_schema.""" diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_ssh.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_ssh.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_ssh.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_ssh.py 2021-05-11 16:34:29.000000000 +0000 @@ -10,6 +10,8 @@ LOG = logging.getLogger(__name__) MODPATH = "cloudinit.config.cc_ssh." +KEY_NAMES_NO_DSA = [name for name in cc_ssh.GENERATE_KEY_NAMES + if name not in 'dsa'] @mock.patch(MODPATH + "ssh_util.setup_user_keys") @@ -25,7 +27,7 @@ } self.test_hostkey_files = [] hostkey_tmpdir = self.tmp_dir() - for key_type in ['dsa', 'ecdsa', 'ed25519', 'rsa']: + for key_type in cc_ssh.GENERATE_KEY_NAMES: key_data = self.test_hostkeys[key_type] filename = 'ssh_host_%s_key.pub' % key_type filepath = os.path.join(hostkey_tmpdir, filename) @@ -223,7 +225,7 @@ cfg = {} expected_call = [self.test_hostkeys[key_type] for key_type - in ['ecdsa', 'ed25519', 'rsa']] + in KEY_NAMES_NO_DSA] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -252,7 +254,7 @@ cfg = {'ssh_publish_hostkeys': {'enabled': True}} expected_call = [self.test_hostkeys[key_type] for key_type - in ['ecdsa', 'ed25519', 'rsa']] + in KEY_NAMES_NO_DSA] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -339,7 +341,65 @@ cfg = {'ssh_publish_hostkeys': {'enabled': True, 'blacklist': []}} expected_call = [self.test_hostkeys[key_type] for key_type - in ['dsa', 'ecdsa', 'ed25519', 'rsa']] + in cc_ssh.GENERATE_KEY_NAMES] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) + + @mock.patch(MODPATH + "ug_util.normalize_users_groups") + @mock.patch(MODPATH + "util.write_file") + def test_handle_ssh_keys_in_cfg(self, m_write_file, m_nug, m_setup_keys): + """Test handle with ssh keys and certificate.""" + # Populate a config dictionary to pass to handle() as well + # as the expected file-writing calls. + cfg = {"ssh_keys": {}} + + expected_calls = [] + for key_type in cc_ssh.GENERATE_KEY_NAMES: + private_name = "{}_private".format(key_type) + public_name = "{}_public".format(key_type) + cert_name = "{}_certificate".format(key_type) + + # Actual key contents don"t have to be realistic + private_value = "{}_PRIVATE_KEY".format(key_type) + public_value = "{}_PUBLIC_KEY".format(key_type) + cert_value = "{}_CERT_KEY".format(key_type) + + cfg["ssh_keys"][private_name] = private_value + cfg["ssh_keys"][public_name] = public_value + cfg["ssh_keys"][cert_name] = cert_value + + expected_calls.extend([ + mock.call( + '/etc/ssh/ssh_host_{}_key'.format(key_type), + private_value, + 384 + ), + mock.call( + '/etc/ssh/ssh_host_{}_key.pub'.format(key_type), + public_value, + 384 + ), + mock.call( + '/etc/ssh/ssh_host_{}_key-cert.pub'.format(key_type), + cert_value, + 384 + ), + mock.call( + '/etc/ssh/sshd_config', + ('HostCertificate /etc/ssh/ssh_host_{}_key-cert.pub' + '\n'.format(key_type)), + preserve_mode=True + ) + ]) + + # Run the handler. + m_nug.return_value = ([], {}) + with mock.patch(MODPATH + 'ssh_util.parse_ssh_config', + return_value=[]): + cc_ssh.handle("name", cfg, self.tmp_cloud(distro='ubuntu'), + LOG, None) + + # Check that all expected output has been done. + for call_ in expected_calls: + self.assertIn(call_, m_write_file.call_args_list) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_ubuntu_advantage.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_ubuntu_advantage.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_ubuntu_advantage.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_ubuntu_advantage.py 2021-05-11 16:34:29.000000000 +0000 @@ -3,7 +3,7 @@ from cloudinit.config.cc_ubuntu_advantage import ( configure_ua, handle, maybe_install_ua_tools, schema) from cloudinit.config.schema import validate_cloudconfig_schema -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import ( CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -26,10 +26,10 @@ super(TestConfigureUA, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_error(self, m_subp): """Errors from ua attach command are raised.""" - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( 'Invalid token SomeToken') with self.assertRaises(RuntimeError) as context_manager: configure_ua(token='SomeToken') @@ -39,7 +39,7 @@ 'Stdout: Invalid token SomeToken\nStderr: -', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_token(self, m_subp): """When token is provided, attach the machine to ua using the token.""" configure_ua(token='SomeToken') @@ -48,7 +48,7 @@ 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_on_service_error(self, m_subp): """all services should be enabled and then any failures raised""" @@ -56,7 +56,7 @@ fail_cmds = [['ua', 'enable', svc] for svc in ['esm', 'cc']] if cmd in fail_cmds and capture: svc = cmd[-1] - raise util.ProcessExecutionError( + raise subp.ProcessExecutionError( 'Invalid {} credentials'.format(svc.upper())) m_subp.side_effect = fake_subp @@ -83,7 +83,7 @@ 'Failure enabling Ubuntu Advantage service(s): "esm", "cc"', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_empty_services(self, m_subp): """When services is an empty list, do not auto-enable attach.""" configure_ua(token='SomeToken', enable=[]) @@ -92,7 +92,7 @@ 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_specific_services(self, m_subp): """When services a list, only enable specific services.""" configure_ua(token='SomeToken', enable=['fips']) @@ -105,7 +105,7 @@ self.logs.getvalue()) @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_string_services(self, m_subp): """When services a string, treat as singleton list and warn""" configure_ua(token='SomeToken', enable='fips') @@ -119,7 +119,7 @@ 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_weird_services(self, m_subp): """When services not string or list, warn but still attach""" configure_ua(token='SomeToken', enable={'deffo': 'wont work'}) @@ -285,7 +285,7 @@ super(TestMaybeInstallUATools, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which): """Do nothing if ubuntu-advantage-tools already exists.""" m_which.return_value = '/usr/bin/ua' # already installed @@ -294,7 +294,7 @@ 'Some apt error') maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_raises_update_errors(self, m_which): """maybe_install_ua_tools logs and raises apt update errors.""" m_which.return_value = None @@ -306,7 +306,7 @@ self.assertEqual('Some apt error', str(context_manager.exception)) self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_raises_install_errors(self, m_which): """maybe_install_ua_tools logs and raises package install errors.""" m_which.return_value = None @@ -320,7 +320,7 @@ self.assertIn( 'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_happy_path(self, m_which): """maybe_install_ua_tools installs ubuntu-advantage-tools.""" m_which.return_value = None diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_ubuntu_drivers.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_ubuntu_drivers.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_ubuntu_drivers.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_ubuntu_drivers.py 2021-05-11 16:34:29.000000000 +0000 @@ -7,7 +7,7 @@ from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) from cloudinit.config import cc_ubuntu_drivers as drivers -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" @@ -16,6 +16,13 @@ "(choose from 'list', 'autoinstall', 'devices', 'debug')\n") +# The tests in this module call helper methods which are decorated with +# mock.patch. pylint doesn't understand that mock.patch passes parameters to +# the decorated function, so it incorrectly reports that we aren't passing +# values for all parameters. Instead of annotating every single call, we +# disable it for the entire module: +# pylint: disable=no-value-for-parameter + class AnyTempScriptAndDebconfFile(object): def __init__(self, tmp_dir, debconf_file): @@ -46,8 +53,8 @@ schema=drivers.schema, strict=True) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_happy_path_taken( self, config, m_which, m_subp, m_tmp): """Positive path test through handle. Package should be installed.""" @@ -73,8 +80,8 @@ self._assert_happy_path_taken(new_config) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_handle_raises_error_if_no_drivers_found( self, m_which, m_subp, m_tmp): """If ubuntu-drivers doesn't install any drivers, raise an error.""" @@ -102,8 +109,8 @@ self.assertIn('ubuntu-drivers found no drivers for installation', self.logs.getvalue()) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_inert_with_config(self, config, m_which, m_subp): """Helper to reduce repetition when testing negative cases""" myCloud = mock.MagicMock() @@ -147,8 +154,8 @@ self.assertEqual(0, m_install_drivers.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=True) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=True) def test_install_drivers_no_install_if_present( self, m_which, m_subp, m_tmp): """If 'ubuntu-drivers' is present, no package install should occur.""" @@ -174,8 +181,8 @@ self.assertEqual(0, pkg_install.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_install_drivers_handles_old_ubuntu_drivers_gracefully( self, m_which, m_subp, m_tmp): """Older ubuntu-drivers versions should emit message and raise error""" @@ -212,8 +219,8 @@ install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123'] @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def test_version_none_uses_latest(self, m_which, m_subp, m_tmp): tdir = self.tmp_dir() debconf_file = os.path.join(tdir, 'nvidia.template') diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_users_groups.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_users_groups.py --- cloud-init-20.1-10-g71af48df/cloudinit/config/tests/test_users_groups.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/config/tests/test_users_groups.py 2021-05-11 16:34:29.000000000 +0000 @@ -39,7 +39,7 @@ cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -65,7 +65,7 @@ cloud = self.tmp_cloud( distro='freebsd', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_fbsd_user.call_args_list, [mock.call('freebsd', groups='wheel', lock_passwd=True, shell='/bin/tcsh'), @@ -86,7 +86,7 @@ cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -107,7 +107,7 @@ cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -146,7 +146,7 @@ cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/almalinux.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/almalinux.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/almalinux.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/almalinux.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,9 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import rhel + + +class Distro(rhel.Distro): + pass + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/alpine.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/alpine.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/alpine.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/alpine.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,188 @@ +# Copyright (C) 2016 Matt Dainty +# Copyright (C) 2020 Dermot Bradley +# +# Author: Matt Dainty +# Author: Dermot Bradley +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import distros +from cloudinit import helpers +from cloudinit import subp +from cloudinit import util + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit.settings import PER_INSTANCE + +NETWORK_FILE_HEADER = """\ +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} + +""" + + +class Distro(distros.Distro): + init_cmd = ['rc-service'] # init scripts + locale_conf_fn = "/etc/profile.d/locale.sh" + network_conf_fn = "/etc/network/interfaces" + renderer_configs = { + "eni": {"eni_path": network_conf_fn, + "eni_header": NETWORK_FILE_HEADER} + } + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + # This will be used to restrict certain + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) + self.default_locale = 'C.UTF-8' + self.osfamily = 'alpine' + cfg['ssh_svcname'] = 'sshd' + + def get_locale(self): + """The default locale for Alpine Linux is different than + cloud-init's DataSource default. + """ + return self.default_locale + + def apply_locale(self, locale, out_fn=None): + # Alpine has limited locale support due to musl library limitations + + if not locale: + locale = self.default_locale + if not out_fn: + out_fn = self.locale_conf_fn + + lines = [ + "#", + "# This file is created by cloud-init once per new instance boot", + "#", + "export CHARSET=UTF-8", + "export LANG=%s" % locale, + "export LC_COLLATE=C", + "", + ] + util.write_file(out_fn, "\n".join(lines), 0o644) + + def install_packages(self, pkglist): + self.update_package_sources() + self.package_command('add', pkgs=pkglist) + + def _write_network_config(self, netconfig): + return self._supported_write_network_config(netconfig) + + def _bring_up_interfaces(self, device_names): + use_all = False + for d in device_names: + if d == 'all': + use_all = True + if use_all: + return distros.Distro._bring_up_interface(self, '-a') + else: + return distros.Distro._bring_up_interfaces(self, device_names) + + def _write_hostname(self, your_hostname, out_fn): + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(your_hostname) + util.write_file(out_fn, str(conf), 0o644) + + def _read_system_hostname(self): + sys_hostname = self._read_hostname(self.hostname_conf_fn) + return (self.hostname_conf_fn, sys_hostname) + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_hostname(self, filename, default=None): + hostname = None + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def _get_localhost_ip(self): + return "127.0.1.1" + + def set_timezone(self, tz): + distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['apk'] + # Redirect output + cmd.append("--quiet") + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + if command: + cmd.append(command) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + subp.subp(cmd, capture=False) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ["update"], freq=PER_INSTANCE) + + @property + def preferred_ntp_clients(self): + """Allow distro to determine the preferred ntp client list""" + if not self._preferred_ntp_clients: + self._preferred_ntp_clients = ['chrony', 'ntp'] + + return self._preferred_ntp_clients + + def shutdown_command(self, mode='poweroff', delay='now', message=None): + # called from cc_power_state_change.load_power_state + # Alpine has halt/poweroff/reboot, with the following specifics: + # - we use them rather than the generic "shutdown" + # - delay is given with "-d [integer]" + # - the integer is in seconds, cannot be "now", and takes no "+" + # - no message is supported (argument ignored, here) + + command = [mode, "-d"] + + # Convert delay from minutes to seconds, as Alpine's + # halt/poweroff/reboot commands take seconds rather than minutes. + if delay == "now": + # Alpine's commands do not understand "now". + command += ['0'] + else: + try: + command.append(str(int(delay) * 60)) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + + return command + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/amazon.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/amazon.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/amazon.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/amazon.py 2021-05-11 16:34:29.000000000 +0000 @@ -12,10 +12,6 @@ from cloudinit.distros import rhel -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - class Distro(rhel.Distro): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/arch.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/arch.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/arch.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/arch.py 2021-05-11 16:34:29.000000000 +0000 @@ -8,6 +8,7 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros import net_util from cloudinit.distros.parsers.hostname import HostnameConf @@ -22,7 +23,7 @@ class Distro(distros.Distro): - locale_conf_fn = "/etc/locale.gen" + locale_gen_fn = "/etc/locale.gen" network_conf_dir = "/etc/netctl" resolve_conf_fn = "/etc/resolv.conf" init_cmd = ['systemctl'] # init scripts @@ -42,16 +43,20 @@ cfg['ssh_svcname'] = 'sshd' def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) - # "" provides trailing newline during join + if out_fn is not None and out_fn != "/etc/locale.conf": + LOG.warning("Invalid locale_configfile %s, only supported " + "value is /etc/locale.conf", out_fn) lines = [ util.make_header(), - 'LANG="%s"' % (locale), + # Hard-coding the charset isn't ideal, but there is no other way. + '%s UTF-8' % (locale), "", ] - util.write_file(out_fn, "\n".join(lines)) + util.write_file(self.locale_gen_fn, "\n".join(lines)) + subp.subp(['locale-gen'], capture=False) + # In the future systemd can handle locale-gen stuff: + # https://github.com/systemd/systemd/pull/9864 + subp.subp(['localectl', 'set-locale', locale], capture=False) def install_packages(self, pkglist): self.update_package_sources() @@ -60,9 +65,9 @@ def _write_network_config(self, netconfig): try: return self._supported_write_network_config(netconfig) - except RendererNotFoundError: + except RendererNotFoundError as e: # Fall back to old _write_network - raise NotImplementedError + raise NotImplementedError from e def _write_network(self, settings): entries = net_util.translate_network(settings) @@ -76,11 +81,11 @@ def _enable_interface(self, device_name): cmd = ['netctl', 'reenable', device_name] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) def _bring_up_interface(self, device_name): @@ -88,12 +93,12 @@ LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -136,6 +141,17 @@ return default return hostname + # hostname (inetutils) isn't installed per default on arch, so we use + # hostnamectl which is installed per default (systemd). + def _apply_hostname(self, hostname): + LOG.debug("Non-persistently setting the system hostname to %s", + hostname) + try: + subp.subp(['hostnamectl', '--transient', 'set-hostname', hostname]) + except subp.ProcessExecutionError: + util.logexc(LOG, "Failed to non-persistently adjust the system " + "hostname to %s", hostname) + def set_timezone(self, tz): distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) @@ -151,6 +167,8 @@ elif args and isinstance(args, list): cmd.extend(args) + if command == "upgrade": + command = "-u" if command: cmd.append(command) @@ -158,7 +176,7 @@ cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, @@ -173,8 +191,8 @@ devs = [] nameservers = [] - resolv_conf = util.target_path(target, resolv_conf) - conf_dir = util.target_path(target, conf_dir) + resolv_conf = subp.target_path(target, resolv_conf) + conf_dir = subp.target_path(target, conf_dir) for (dev, info) in entries.items(): if dev == 'lo': diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/bsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/bsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/bsd.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/bsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,133 @@ +import platform + +from cloudinit import distros +from cloudinit.distros import bsd_utils +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import net +from cloudinit import subp +from cloudinit import util +from .networking import BSDNetworking + +LOG = logging.getLogger(__name__) + + +class BSD(distros.Distro): + networking_cls = BSDNetworking + hostname_conf_fn = '/etc/rc.conf' + rc_conf_fn = "/etc/rc.conf" + + # This differs from the parent Distro class, which has -P for + # poweroff. + shutdown_options_map = {'halt': '-H', 'poweroff': '-p', 'reboot': '-r'} + + # Set in BSD distro subclasses + group_add_cmd_prefix = [] + pkg_cmd_install_prefix = [] + pkg_cmd_remove_prefix = [] + # There is no update/upgrade on OpenBSD + pkg_cmd_update_prefix = None + pkg_cmd_upgrade_prefix = None + + def __init__(self, name, cfg, paths): + super().__init__(name, cfg, paths) + # This will be used to restrict certain + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) + cfg['ssh_svcname'] = 'sshd' + self.osfamily = platform.system().lower() + + def _read_system_hostname(self): + sys_hostname = self._read_hostname(self.hostname_conf_fn) + return (self.hostname_conf_fn, sys_hostname) + + def _read_hostname(self, filename, default=None): + return bsd_utils.get_rc_config_value('hostname') + + def _get_add_member_to_group_cmd(self, member_name, group_name): + raise NotImplementedError('Return list cmd to add member to group') + + def _write_hostname(self, hostname, filename): + bsd_utils.set_rc_config_value('hostname', hostname, fn='/etc/rc.conf') + + def create_group(self, name, members=None): + if util.is_group(name): + LOG.warning("Skipping creation of existing group '%s'", name) + else: + group_add_cmd = self.group_add_cmd_prefix + [name] + try: + subp.subp(group_add_cmd) + LOG.info("Created new group %s", name) + except Exception: + util.logexc(LOG, "Failed to create group %s", name) + + if not members: + members = [] + for member in members: + if not util.is_user(member): + LOG.warning("Unable to add group member '%s' to group '%s'" + "; user does not exist.", member, name) + continue + try: + subp.subp(self._get_add_member_to_group_cmd(member, name)) + LOG.info("Added user '%s' to group '%s'", member, name) + except Exception: + util.logexc(LOG, "Failed to add user '%s' to group '%s'", + member, name) + + def generate_fallback_config(self): + nconf = {'config': [], 'version': 1} + for mac, name in net.get_interfaces_by_mac().items(): + nconf['config'].append( + {'type': 'physical', 'name': name, + 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]}) + return nconf + + def install_packages(self, pkglist): + self.update_package_sources() + self.package_command('install', pkgs=pkglist) + + def _get_pkg_cmd_environ(self): + """Return environment vars used in *BSD package_command operations""" + raise NotImplementedError('BSD subclasses return a dict of env vars') + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + if command == 'install': + cmd = self.pkg_cmd_install_prefix + elif command == 'remove': + cmd = self.pkg_cmd_remove_prefix + elif command == 'update': + if not self.pkg_cmd_update_prefix: + return + cmd = self.pkg_cmd_update_prefix + elif command == 'upgrade': + if not self.pkg_cmd_upgrade_prefix: + return + cmd = self.pkg_cmd_upgrade_prefix + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + subp.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False) + + def _write_network_config(self, netconfig): + return self._supported_write_network_config(netconfig) + + def set_timezone(self, tz): + distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) + + def apply_locale(self, locale, out_fn=None): + LOG.debug('Cannot set the locale.') + + def apply_network_config_names(self, netconfig): + LOG.debug('Cannot rename network interface.') diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/bsd_utils.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/bsd_utils.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/bsd_utils.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/bsd_utils.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,50 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import shlex + +from cloudinit import util + +# On NetBSD, /etc/rc.conf comes with a if block: +# if [ -r /etc/defaults/rc.conf ]; then +# as a consequence, the file is not a regular key/value list +# anymore and we cannot use cloudinit.distros.parsers.sys_conf +# The module comes with a more naive parser, but is able to +# preserve these if blocks. + + +def _unquote(value): + if value[0] == value[-1] and value[0] in ['"', "'"]: + return value[1:-1] + return value + + +def get_rc_config_value(key, fn='/etc/rc.conf'): + key_prefix = '{}='.format(key) + for line in util.load_file(fn).splitlines(): + if line.startswith(key_prefix): + value = line.replace(key_prefix, '') + return _unquote(value) + + +def set_rc_config_value(key, value, fn='/etc/rc.conf'): + lines = [] + done = False + value = shlex.quote(value) + original_content = util.load_file(fn) + for line in original_content.splitlines(): + if '=' in line: + k, v = line.split('=', 1) + if k == key: + v = value + done = True + lines.append('='.join([k, v])) + else: + lines.append(line) + if not done: + lines.append('='.join([key, value])) + new_content = '\n'.join(lines) + '\n' + if new_content != original_content: + util.write_file(fn, new_content) + + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/centos.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/centos.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/centos.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/centos.py 2021-05-11 16:34:29.000000000 +0000 @@ -1,9 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.distros import rhel -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) class Distro(rhel.Distro): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/debian.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/debian.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/debian.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/debian.py 2021-05-11 16:34:29.000000000 +0000 @@ -13,6 +13,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -197,7 +198,7 @@ # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, msg="apt-%s [%s]" % (command, ' '.join(cmd)), - func=util.subp, + func=subp.subp, args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): @@ -214,7 +215,7 @@ if (util.is_true(mode) or (str(mode).lower() == "auto" and cmd[0] and - util.which(cmd[0]))): + subp.which(cmd[0]))): return cmd else: return [] @@ -269,7 +270,7 @@ """Update system locale config""" LOG.debug('Updating %s with locale setting %s=%s', sys_path, keyname, locale) - util.subp( + subp.subp( ['update-locale', '--locale-file=' + sys_path, '%s=%s' % (keyname, locale)], capture=False) @@ -291,7 +292,7 @@ # finally, trigger regeneration LOG.debug('Generating locales for %s', locale) - util.subp(['locale-gen', locale], capture=False) + subp.subp(['locale-gen', locale], capture=False) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/fedora.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/fedora.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/fedora.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/fedora.py 2021-05-11 16:34:29.000000000 +0000 @@ -10,10 +10,6 @@ from cloudinit.distros import rhel -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - class Distro(rhel.Distro): pass diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/freebsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/freebsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/freebsd.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/freebsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -8,80 +8,29 @@ import re from io import StringIO -from cloudinit import distros -from cloudinit import helpers +import cloudinit.distros.bsd from cloudinit import log as logging -from cloudinit import net -from cloudinit import ssh_util +from cloudinit import subp from cloudinit import util -from cloudinit.distros import rhel_util from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) -class Distro(distros.Distro): +class Distro(cloudinit.distros.bsd.BSD): usr_lib_exec = '/usr/local/lib' - rc_conf_fn = "/etc/rc.conf" login_conf_fn = '/etc/login.conf' login_conf_fn_bak = '/etc/login.conf.orig' ci_sudoers_fn = '/usr/local/etc/sudoers.d/90-cloud-init-users' - hostname_conf_fn = '/etc/rc.conf' + group_add_cmd_prefix = ['pw', 'group', 'add'] + pkg_cmd_install_prefix = ["pkg", "install"] + pkg_cmd_remove_prefix = ["pkg", "remove"] + pkg_cmd_update_prefix = ["pkg", "update"] + pkg_cmd_upgrade_prefix = ["pkg", "upgrade"] + prefer_fqdn = True # See rc.conf(5) in FreeBSD - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'freebsd' - cfg['ssh_svcname'] = 'sshd' - - def _select_hostname(self, hostname, fqdn): - # Should be FQDN if available. See rc.conf(5) in FreeBSD - if fqdn: - return fqdn - return hostname - - def _read_system_hostname(self): - sys_hostname = self._read_hostname(self.hostname_conf_fn) - return (self.hostname_conf_fn, sys_hostname) - - def _read_hostname(self, filename, default=None): - (_exists, contents) = rhel_util.read_sysconfig_file(filename) - if contents.get('hostname'): - return contents['hostname'] - else: - return default - - def _write_hostname(self, hostname, filename): - rhel_util.update_sysconfig_file(filename, {'hostname': hostname}) - - def create_group(self, name, members): - group_add_cmd = ['pw', 'group', 'add', name] - if util.is_group(name): - LOG.warning("Skipping creation of existing group '%s'", name) - else: - try: - util.subp(group_add_cmd) - LOG.info("Created new group %s", name) - except Exception: - util.logexc(LOG, "Failed to create group %s", name) - raise - if not members: - members = [] - - for member in members: - if not util.is_user(member): - LOG.warning("Unable to add group member '%s' to group '%s'" - "; user does not exist.", member, name) - continue - try: - util.subp(['pw', 'usermod', '-n', name, '-G', member]) - LOG.info("Added user '%s' to group '%s'", member, name) - except Exception: - util.logexc(LOG, "Failed to add user '%s' to group '%s'", - member, name) + def _get_add_member_to_group_cmd(self, member_name, group_name): + return ['pw', 'usermod', '-n', member_name, '-G', group_name] def add_user(self, name, **kwargs): if util.is_user(name): @@ -125,7 +74,7 @@ # Run the command LOG.info("Adding user %s", name) try: - util.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) + subp.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) except Exception: util.logexc(LOG, "Failed to create user %s", name) raise @@ -137,7 +86,7 @@ def expire_passwd(self, user): try: - util.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) + subp.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise @@ -149,7 +98,7 @@ hash_opt = "-h" try: - util.subp(['pw', 'usermod', user, hash_opt, '0'], + subp.subp(['pw', 'usermod', user, hash_opt, '0'], data=passwd, logstring="chpasswd for %s" % user) except Exception: util.logexc(LOG, "Failed to set password for %s", user) @@ -157,45 +106,13 @@ def lock_passwd(self, name): try: - util.subp(['pw', 'usermod', name, '-h', '-']) + subp.subp(['pw', 'usermod', name, '-h', '-']) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise - def create_user(self, name, **kwargs): - self.add_user(name, **kwargs) - - # Set password if plain-text password provided and non-empty - if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: - self.set_passwd(name, kwargs['plain_text_passwd']) - - # Default locking down the account. 'lock_passwd' defaults to True. - # lock account unless lock_password is False. - if kwargs.get('lock_passwd', True): - self.lock_passwd(name) - - # Configure sudo access - if 'sudo' in kwargs and kwargs['sudo'] is not False: - self.write_sudo_rules(name, kwargs['sudo']) - - # Import SSH keys - if 'ssh_authorized_keys' in kwargs: - keys = set(kwargs['ssh_authorized_keys']) or [] - ssh_util.setup_user_keys(keys, name, options=None) - - def generate_fallback_config(self): - nconf = {'config': [], 'version': 1} - for mac, name in net.get_interfaces_by_mac().items(): - nconf['config'].append( - {'type': 'physical', 'name': name, - 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]}) - return nconf - - def _write_network_config(self, netconfig): - return self._supported_write_network_config(netconfig) - def apply_locale(self, locale, out_fn=None): - # Adjust the locals value to the new value + # Adjust the locales value to the new value newconf = StringIO() for line in util.load_file(self.login_conf_fn).splitlines(): newconf.write(re.sub(r'^default:', @@ -210,8 +127,8 @@ try: LOG.debug("Running cap_mkdb for %s", locale) - util.subp(['cap_mkdb', self.login_conf_fn]) - except util.ProcessExecutionError: + subp.subp(['cap_mkdb', self.login_conf_fn]) + except subp.ProcessExecutionError: # cap_mkdb failed, so restore the backup. util.logexc(LOG, "Failed to apply locale %s", locale) try: @@ -225,39 +142,17 @@ # /etc/rc.conf a line with the following format: # ifconfig_OLDNAME_name=NEWNAME # FreeBSD network script will rename the interface automatically. - return - - def install_packages(self, pkglist): - self.update_package_sources() - self.package_command('install', pkgs=pkglist) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] + pass + def _get_pkg_cmd_environ(self): + """Return environment vars used in *BSD package_command operations""" e = os.environ.copy() e['ASSUME_ALWAYS_YES'] = 'YES' - - cmd = ['pkg'] - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - if command: - cmd.append(command) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=e, capture=False) - - def set_timezone(self, tz): - distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) + return e def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ["update"], freq=PER_INSTANCE) + self._runner.run( + "update-sources", self.package_command, + ["update"], freq=PER_INSTANCE) # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/gentoo.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/gentoo.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/gentoo.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/gentoo.py 2021-05-11 16:34:29.000000000 +0000 @@ -9,6 +9,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import net_util @@ -39,7 +40,7 @@ def apply_locale(self, locale, out_fn=None): if not out_fn: out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) + subp.subp(['locale-gen', '-G', locale], capture=False) # "" provides trailing newline during join lines = [ util.make_header(), @@ -94,11 +95,11 @@ cmd = ['rc-update', 'add', 'net.{name}'.format(name=dev), 'default'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) @@ -119,12 +120,12 @@ LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -137,11 +138,11 @@ # Grab device names from init scripts cmd = ['ls', '/etc/init.d/net.*'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False devices = [x.split('.')[2] for x in _out.split(' ')] @@ -159,10 +160,12 @@ pass if not conf: conf = HostnameConf('') - conf.set_hostname(your_hostname) - gentoo_hostname_config = 'hostname="%s"' % conf - gentoo_hostname_config = gentoo_hostname_config.replace('\n', '') - util.write_file(out_fn, gentoo_hostname_config, 0o644) + + # Many distro's format is the hostname by itself, and that is the + # way HostnameConf works but gentoo expects it to be in + # hostname="the-actual-hostname" + conf.set_hostname('hostname="%s"' % your_hostname) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) @@ -208,7 +211,7 @@ cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/__init__.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/__init__.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/__init__.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/__init__.py 2021-05-11 16:34:29.000000000 +0000 @@ -13,6 +13,8 @@ import os import re import stat +import string +import urllib.parse from io import StringIO from cloudinit import importer @@ -21,11 +23,17 @@ from cloudinit.net import eni from cloudinit.net import network_state from cloudinit.net import renderers +from cloudinit import persistence from cloudinit import ssh_util from cloudinit import type_utils +from cloudinit import subp from cloudinit import util +from cloudinit.features import \ + ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES + from cloudinit.distros.parsers import hosts +from .networking import LinuxNetworking # Used when a cloud-config module can be run on all cloud-init distibutions. @@ -33,12 +41,13 @@ ALL_DISTROS = 'all' OSFAMILIES = { + 'alpine': ['alpine'], + 'arch': ['arch'], 'debian': ['debian', 'ubuntu'], - 'redhat': ['amazon', 'centos', 'fedora', 'rhel'], - 'gentoo': ['gentoo'], 'freebsd': ['freebsd'], + 'gentoo': ['gentoo'], + 'redhat': ['almalinux', 'amazon', 'centos', 'fedora', 'rhel'], 'suse': ['opensuse', 'sles'], - 'arch': ['arch'], } LOG = logging.getLogger(__name__) @@ -50,8 +59,11 @@ # Default NTP Client Configurations PREFERRED_NTP_CLIENTS = ['chrony', 'systemd-timesyncd', 'ntp', 'ntpdate'] +# Letters/Digits/Hyphen characters, for use in domain name validation +LDH_ASCII_CHARS = string.ascii_letters + string.digits + "-" -class Distro(metaclass=abc.ABCMeta): + +class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): usr_lib_exec = "/usr/lib" hosts_fn = "/etc/hosts" @@ -61,11 +73,31 @@ init_cmd = ['service'] # systemctl, service etc renderer_configs = {} _preferred_ntp_clients = None + networking_cls = LinuxNetworking + # This is used by self.shutdown_command(), and can be overridden in + # subclasses + shutdown_options_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} + + _ci_pkl_version = 1 + prefer_fqdn = False def __init__(self, name, cfg, paths): self._paths = paths self._cfg = cfg self.name = name + self.networking = self.networking_cls() + + def _unpickle(self, ci_pkl_version: int) -> None: + """Perform deserialization fixes for Distro.""" + if "networking" not in self.__dict__ or not self.networking.__dict__: + # This is either a Distro pickle with no networking attribute OR + # this is a Distro pickle with a networking attribute but from + # before ``Networking`` had any state (meaning that + # Networking.__setstate__ will not be called). In either case, we + # want to ensure that `self.networking` is freshly-instantiated: + # either because it isn't present at all, or because it will be + # missing expected instance state otherwise. + self.networking = self.networking_cls() @abc.abstractmethod def install_packages(self, pkglist): @@ -100,6 +132,9 @@ def get_option(self, opt_name, default=None): return self._cfg.get(opt_name, default) + def set_option(self, opt_name, value=None): + self._cfg[opt_name] = value + def set_hostname(self, hostname, fqdn=None): writeable_hostname = self._select_hostname(hostname, fqdn) self._write_hostname(writeable_hostname, self.hostname_conf_fn) @@ -220,14 +255,17 @@ LOG.debug("Non-persistently setting the system hostname to %s", hostname) try: - util.subp(['hostname', hostname]) - except util.ProcessExecutionError: + subp.subp(['hostname', hostname]) + except subp.ProcessExecutionError: util.logexc(LOG, "Failed to non-persistently adjust the system " "hostname to %s", hostname) def _select_hostname(self, hostname, fqdn): # Prefer the short hostname over the long # fully qualified domain name + if util.get_cfg_option_bool(self._cfg, "prefer_fqdn_over_hostname", + self.prefer_fqdn) and fqdn: + return fqdn if not hostname: return fqdn return hostname @@ -237,8 +275,9 @@ distros = [] for family in family_list: if family not in OSFAMILIES: - raise ValueError("No distibutions found for osfamily %s" - % (family)) + raise ValueError( + "No distributions found for osfamily {}".format(family) + ) distros.extend(OSFAMILIES[family]) return distros @@ -356,12 +395,12 @@ LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -380,6 +419,9 @@ def add_user(self, name, **kwargs): """ Add a user to the system using standard GNU tools + + This should be overriden on distros where useradd is not desirable or + not available. """ # XXX need to make add_user idempotent somehow as we # still want to add groups or modify SSH keys on pre-existing @@ -475,7 +517,7 @@ # Run the command LOG.debug("Adding user %s", name) try: - util.subp(useradd_cmd, logstring=log_useradd_cmd) + subp.subp(useradd_cmd, logstring=log_useradd_cmd) except Exception as e: util.logexc(LOG, "Failed to create user %s", name) raise e @@ -495,7 +537,7 @@ # Run the command LOG.debug("Adding snap user %s", name) try: - (out, err) = util.subp(create_user_cmd, logstring=create_user_cmd, + (out, err) = subp.subp(create_user_cmd, logstring=create_user_cmd, capture=True) LOG.debug("snap create-user returned: %s:%s", out, err) jobj = util.load_json(out) @@ -508,9 +550,22 @@ def create_user(self, name, **kwargs): """ - Creates users for the system using the GNU passwd tools. This - will work on an GNU system. This should be overriden on - distros where useradd is not desirable or not available. + Creates or partially updates the ``name`` user in the system. + + This defers the actual user creation to ``self.add_user`` or + ``self.add_snap_user``, and most of the keys in ``kwargs`` will be + processed there if and only if the user does not already exist. + + Once the existence of the ``name`` user has been ensured, this method + then processes these keys (for both just-created and pre-existing + users): + + * ``plain_text_passwd`` + * ``hashed_passwd`` + * ``lock_passwd`` + * ``sudo`` + * ``ssh_authorized_keys`` + * ``ssh_redirect_user`` """ # Add a snap user, if requested @@ -577,20 +632,21 @@ # passwd must use short '-l' due to SLES11 lacking long form '--lock' lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: - cmd = next(l for l in lock_tools if util.which(l[0])) - except StopIteration: + cmd = next(tool for tool in lock_tools if subp.which(tool[0])) + except StopIteration as e: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " - " Tried: %s.") % (name, [c[0] for c in lock_tools])) + " Tried: %s.") % (name, [c[0] for c in lock_tools]) + ) from e try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: util.logexc(LOG, 'Failed to disable password for user %s', name) raise e def expire_passwd(self, user): try: - util.subp(['passwd', '--expire', user]) + subp.subp(['passwd', '--expire', user]) except Exception as e: util.logexc(LOG, "Failed to set 'expire' for %s", user) raise e @@ -606,7 +662,7 @@ cmd.append('-e') try: - util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) + subp.subp(cmd, pass_string, logstring="chpasswd for %s" % user) except Exception as e: util.logexc(LOG, "Failed to set password for %s", user) raise e @@ -624,7 +680,7 @@ found_include = False for line in sudoers_contents.splitlines(): line = line.strip() - include_match = re.search(r"^#includedir\s+(.*)$", line) + include_match = re.search(r"^[#|@]includedir\s+(.*)$", line) if not include_match: continue included_dir = include_match.group(1).strip() @@ -703,7 +759,7 @@ LOG.warning("Skipping creation of existing group '%s'", name) else: try: - util.subp(group_add_cmd) + subp.subp(group_add_cmd) LOG.info("Created new group %s", name) except Exception: util.logexc(LOG, "Failed to create group %s", name) @@ -716,9 +772,130 @@ "; user does not exist.", member, name) continue - util.subp(['usermod', '-a', '-G', name, member]) + subp.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'", member, name) + def shutdown_command(self, *, mode, delay, message): + # called from cc_power_state_change.load_power_state + command = ["shutdown", self.shutdown_options_map[mode]] + try: + if delay != "now": + delay = "+%d" % int(delay) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + args = command + [delay] + if message: + args.append(message) + return args + + +def _apply_hostname_transformations_to_url(url: str, transformations: list): + """ + Apply transformations to a URL's hostname, return transformed URL. + + This is a separate function because unwrapping and rewrapping only the + hostname portion of a URL is complex. + + :param url: + The URL to operate on. + :param transformations: + A list of ``(str) -> Optional[str]`` functions, which will be applied + in order to the hostname portion of the URL. If any function + (regardless of ordering) returns None, ``url`` will be returned without + any modification. + + :return: + A string whose value is ``url`` with the hostname ``transformations`` + applied, or ``None`` if ``url`` is unparseable. + """ + try: + parts = urllib.parse.urlsplit(url) + except ValueError: + # If we can't even parse the URL, we shouldn't use it for anything + return None + new_hostname = parts.hostname + if new_hostname is None: + # The URL given doesn't have a hostname component, so (a) we can't + # transform it, and (b) it won't work as a mirror; return None. + return None + + for transformation in transformations: + new_hostname = transformation(new_hostname) + if new_hostname is None: + # If a transformation returns None, that indicates we should abort + # processing and return `url` unmodified + return url + + new_netloc = new_hostname + if parts.port is not None: + new_netloc = "{}:{}".format(new_netloc, parts.port) + return urllib.parse.urlunsplit(parts._replace(netloc=new_netloc)) + + +def _sanitize_mirror_url(url: str): + """ + Given a mirror URL, replace or remove any invalid URI characters. + + This performs the following actions on the URL's hostname: + * Checks if it is an IP address, returning the URL immediately if it is + * Converts it to its IDN form (see below for details) + * Replaces any non-Letters/Digits/Hyphen (LDH) characters in it with + hyphens + * TODO: Remove any leading/trailing hyphens from each domain name label + + Before we replace any invalid domain name characters, we first need to + ensure that any valid non-ASCII characters in the hostname will not be + replaced, by ensuring the hostname is in its Internationalized domain name + (IDN) representation (see RFC 5890). This conversion has to be applied to + the whole hostname (rather than just the substitution variables), because + the Punycode algorithm used by IDNA transcodes each part of the hostname as + a whole string (rather than encoding individual characters). It cannot be + applied to the whole URL, because (a) the Punycode algorithm expects to + operate on domain names so doesn't output a valid URL, and (b) non-ASCII + characters in non-hostname parts of the URL aren't encoded via Punycode. + + To put this in RFC 5890's terminology: before we remove or replace any + characters from our domain name (which we do to ensure that each label is a + valid LDH Label), we first ensure each label is in its A-label form. + + (Note that Python's builtin idna encoding is actually IDNA2003, not + IDNA2008. This changes the specifics of how some characters are encoded to + ASCII, but doesn't affect the logic here.) + + :param url: + The URL to operate on. + + :return: + A sanitized version of the URL, which will have been IDNA encoded if + necessary, or ``None`` if the generated string is not a parseable URL. + """ + # Acceptable characters are LDH characters, plus "." to separate each label + acceptable_chars = LDH_ASCII_CHARS + "." + transformations = [ + # This is an IP address, not a hostname, so no need to apply the + # transformations + lambda hostname: None if net.is_ip_address(hostname) else hostname, + + # Encode with IDNA to get the correct characters (as `bytes`), then + # decode with ASCII so we return a `str` + lambda hostname: hostname.encode('idna').decode('ascii'), + + # Replace any unacceptable characters with "-" + lambda hostname: ''.join( + c if c in acceptable_chars else "-" for c in hostname + ), + + # Drop leading/trailing hyphens from each part of the hostname + lambda hostname: '.'.join( + part.strip('-') for part in hostname.split('.') + ), + ] + + return _apply_hostname_transformations_to_url(url, transformations) + def _get_package_mirror_info(mirror_info, data_source=None, mirror_filter=util.search_for_mirror): @@ -735,7 +912,12 @@ # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b) # the region is us-east-1. so region = az[0:-1] if _EC2_AZ_RE.match(data_source.availability_zone): - subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] + ec2_region = data_source.availability_zone[0:-1] + + if ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES: + subst['ec2_region'] = "%s" % ec2_region + elif data_source.platform_type == "ec2": + subst['ec2_region'] = "%s" % ec2_region if data_source and data_source.region: subst['region'] = data_source.region @@ -748,9 +930,13 @@ mirrors = [] for tmpl in searchlist: try: - mirrors.append(tmpl % subst) + mirror = tmpl % subst except KeyError: - pass + continue + + mirror = _sanitize_mirror_url(mirror) + if mirror is not None: + mirrors.append(mirror) found = mirror_filter(mirrors) if found: diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/netbsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/netbsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/netbsd.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/netbsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,159 @@ +# Copyright (C) 2019-2020 Gonéri Le Bouder +# +# This file is part of cloud-init. See LICENSE file for license information. + +import crypt +import os +import platform + +import cloudinit.distros.bsd +from cloudinit import log as logging +from cloudinit import subp +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class NetBSD(cloudinit.distros.bsd.BSD): + """ + Distro subclass for NetBSD. + + (N.B. OpenBSD inherits from this class.) + """ + + ci_sudoers_fn = '/usr/pkg/etc/sudoers.d/90-cloud-init-users' + group_add_cmd_prefix = ["groupadd"] + + def __init__(self, name, cfg, paths): + super().__init__(name, cfg, paths) + if os.path.exists("/usr/pkg/bin/pkgin"): + self.pkg_cmd_install_prefix = ['pkgin', '-y', 'install'] + self.pkg_cmd_remove_prefix = ['pkgin', '-y', 'remove'] + self.pkg_cmd_update_prefix = ['pkgin', '-y', 'update'] + self.pkg_cmd_upgrade_prefix = ['pkgin', '-y', 'full-upgrade'] + else: + self.pkg_cmd_install_prefix = ['pkg_add', '-U'] + self.pkg_cmd_remove_prefix = ['pkg_delete'] + + def _get_add_member_to_group_cmd(self, member_name, group_name): + return ['usermod', '-G', group_name, member_name] + + def add_user(self, name, **kwargs): + if util.is_user(name): + LOG.info("User %s already exists, skipping.", name) + return False + + adduser_cmd = ['useradd'] + log_adduser_cmd = ['useradd'] + + adduser_opts = { + "homedir": '-d', + "gecos": '-c', + "primary_group": '-g', + "groups": '-G', + "shell": '-s', + } + adduser_flags = { + "no_user_group": '--no-user-group', + "system": '--system', + "no_log_init": '--no-log-init', + } + + for key, val in kwargs.items(): + if key in adduser_opts and val and isinstance(val, str): + adduser_cmd.extend([adduser_opts[key], val]) + + elif key in adduser_flags and val: + adduser_cmd.append(adduser_flags[key]) + log_adduser_cmd.append(adduser_flags[key]) + + if 'no_create_home' not in kwargs or 'system' not in kwargs: + adduser_cmd += ['-m'] + log_adduser_cmd += ['-m'] + + adduser_cmd += [name] + log_adduser_cmd += [name] + + # Run the command + LOG.info("Adding user %s", name) + try: + subp.subp(adduser_cmd, logstring=log_adduser_cmd) + except Exception: + util.logexc(LOG, "Failed to create user %s", name) + raise + # Set the password if it is provided + # For security consideration, only hashed passwd is assumed + passwd_val = kwargs.get('passwd', None) + if passwd_val is not None: + self.set_passwd(name, passwd_val, hashed=True) + + def set_passwd(self, user, passwd, hashed=False): + if hashed: + hashed_pw = passwd + elif not hasattr(crypt, 'METHOD_BLOWFISH'): + # crypt.METHOD_BLOWFISH comes with Python 3.7 which is available + # on NetBSD 7 and 8. + LOG.error(( + 'Cannot set non-encrypted password for user %s. ' + 'Python >= 3.7 is required.'), user) + return + else: + method = crypt.METHOD_BLOWFISH # pylint: disable=E1101 + hashed_pw = crypt.crypt( + passwd, + crypt.mksalt(method) + ) + + try: + subp.subp(['usermod', '-p', hashed_pw, user]) + except Exception: + util.logexc(LOG, "Failed to set password for %s", user) + raise + self.unlock_passwd(user) + + def force_passwd_change(self, user): + try: + subp.subp(['usermod', '-F', user]) + except Exception: + util.logexc(LOG, "Failed to set pw expiration for %s", user) + raise + + def lock_passwd(self, name): + try: + subp.subp(['usermod', '-C', 'yes', name]) + except Exception: + util.logexc(LOG, "Failed to lock user %s", name) + raise + + def unlock_passwd(self, name): + try: + subp.subp(['usermod', '-C', 'no', name]) + except Exception: + util.logexc(LOG, "Failed to unlock user %s", name) + raise + + def apply_locale(self, locale, out_fn=None): + LOG.debug('Cannot set the locale.') + + def apply_network_config_names(self, netconfig): + LOG.debug('NetBSD cannot rename network interface.') + + def _get_pkg_cmd_environ(self): + """Return env vars used in NetBSD package_command operations""" + os_release = platform.release() + os_arch = platform.machine() + e = os.environ.copy() + e['PKG_PATH'] = ( + 'http://cdn.netbsd.org/pub/pkgsrc/' + 'packages/NetBSD/%s/%s/All' + ) % (os_arch, os_release) + return e + + def update_package_sources(self): + pass + + +class Distro(NetBSD): + pass + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/networking.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/networking.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/networking.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/networking.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,231 @@ +import abc +import logging +import os + +from cloudinit import subp +from cloudinit import net, util + + +LOG = logging.getLogger(__name__) + + +# Type aliases (https://docs.python.org/3/library/typing.html#type-aliases), +# used to make the signatures of methods a little clearer +DeviceName = str +NetworkConfig = dict + + +class Networking(metaclass=abc.ABCMeta): + """The root of the Networking hierarchy in cloud-init. + + This is part of an ongoing refactor in the cloud-init codebase, for more + details see "``cloudinit.net`` -> ``cloudinit.distros.networking`` + Hierarchy" in HACKING.rst for full details. + """ + + def __init__(self): + self.blacklist_drivers = None + + def _get_current_rename_info(self) -> dict: + return net._get_current_rename_info() + + def _rename_interfaces(self, renames: list, *, current_info=None) -> None: + return net._rename_interfaces(renames, current_info=current_info) + + def apply_network_config_names(self, netcfg: NetworkConfig) -> None: + return net.apply_network_config_names(netcfg) + + def device_devid(self, devname: DeviceName): + return net.device_devid(devname) + + def device_driver(self, devname: DeviceName): + return net.device_driver(devname) + + def extract_physdevs(self, netcfg: NetworkConfig) -> list: + return net.extract_physdevs(netcfg) + + def find_fallback_nic(self, *, blacklist_drivers=None): + return net.find_fallback_nic(blacklist_drivers=blacklist_drivers) + + def generate_fallback_config( + self, *, blacklist_drivers=None, config_driver: bool = False + ): + return net.generate_fallback_config( + blacklist_drivers=blacklist_drivers, config_driver=config_driver + ) + + def get_devicelist(self) -> list: + return net.get_devicelist() + + def get_ib_hwaddrs_by_interface(self) -> dict: + return net.get_ib_hwaddrs_by_interface() + + def get_ib_interface_hwaddr( + self, devname: DeviceName, ethernet_format: bool + ): + return net.get_ib_interface_hwaddr(devname, ethernet_format) + + def get_interface_mac(self, devname: DeviceName): + return net.get_interface_mac(devname) + + def get_interfaces(self) -> list: + return net.get_interfaces() + + def get_interfaces_by_mac(self) -> dict: + return net.get_interfaces_by_mac( + blacklist_drivers=self.blacklist_drivers) + + def get_master(self, devname: DeviceName): + return net.get_master(devname) + + def interface_has_own_mac( + self, devname: DeviceName, *, strict: bool = False + ) -> bool: + return net.interface_has_own_mac(devname, strict=strict) + + def is_bond(self, devname: DeviceName) -> bool: + return net.is_bond(devname) + + def is_bridge(self, devname: DeviceName) -> bool: + return net.is_bridge(devname) + + @abc.abstractmethod + def is_physical(self, devname: DeviceName) -> bool: + """ + Is ``devname`` a physical network device? + + Examples of non-physical network devices: bonds, bridges, tunnels, + loopback devices. + """ + + def is_renamed(self, devname: DeviceName) -> bool: + return net.is_renamed(devname) + + def is_up(self, devname: DeviceName) -> bool: + return net.is_up(devname) + + def is_vlan(self, devname: DeviceName) -> bool: + return net.is_vlan(devname) + + def master_is_bridge_or_bond(self, devname: DeviceName) -> bool: + return net.master_is_bridge_or_bond(devname) + + @abc.abstractmethod + def settle(self, *, exists=None) -> None: + """Wait for device population in the system to complete. + + :param exists: + An optional optimisation. If given, only perform as much of the + settle process as is required for the given DeviceName to be + present in the system. (This may include skipping the settle + process entirely, if the device already exists.) + :type exists: Optional[DeviceName] + """ + + def wait_for_physdevs( + self, netcfg: NetworkConfig, *, strict: bool = True + ) -> None: + """Wait for all the physical devices in `netcfg` to exist on the system + + Specifically, this will call `self.settle` 5 times, and check after + each one if the physical devices are now present in the system. + + :param netcfg: + The NetworkConfig from which to extract physical devices to wait + for. + :param strict: + Raise a `RuntimeError` if any physical devices are not present + after waiting. + """ + physdevs = self.extract_physdevs(netcfg) + + # set of expected iface names and mac addrs + expected_ifaces = dict([(iface[0], iface[1]) for iface in physdevs]) + expected_macs = set(expected_ifaces.keys()) + + # set of current macs + present_macs = self.get_interfaces_by_mac().keys() + + # compare the set of expected mac address values to + # the current macs present; we only check MAC as cloud-init + # has not yet renamed interfaces and the netcfg may include + # such renames. + for _ in range(0, 5): + if expected_macs.issubset(present_macs): + LOG.debug("net: all expected physical devices present") + return + + missing = expected_macs.difference(present_macs) + LOG.debug("net: waiting for expected net devices: %s", missing) + for mac in missing: + # trigger a settle, unless this interface exists + devname = expected_ifaces[mac] + msg = "Waiting for settle or {} exists".format(devname) + util.log_time( + LOG.debug, + msg, + func=self.settle, + kwargs={"exists": devname}, + ) + + # update present_macs after settles + present_macs = self.get_interfaces_by_mac().keys() + + msg = "Not all expected physical devices present: %s" % missing + LOG.warning(msg) + if strict: + raise RuntimeError(msg) + + @abc.abstractmethod + def try_set_link_up(self, devname: DeviceName) -> bool: + """Try setting the link to up explicitly and return if it is up.""" + + +class BSDNetworking(Networking): + """Implementation of networking functionality shared across BSDs.""" + + def is_physical(self, devname: DeviceName) -> bool: + raise NotImplementedError() + + def settle(self, *, exists=None) -> None: + """BSD has no equivalent to `udevadm settle`; noop.""" + + def try_set_link_up(self, devname: DeviceName) -> bool: + raise NotImplementedError() + + +class LinuxNetworking(Networking): + """Implementation of networking functionality common to Linux distros.""" + + def get_dev_features(self, devname: DeviceName) -> str: + return net.get_dev_features(devname) + + def has_netfail_standby_feature(self, devname: DeviceName) -> bool: + return net.has_netfail_standby_feature(devname) + + def is_netfailover(self, devname: DeviceName) -> bool: + return net.is_netfailover(devname) + + def is_netfail_master(self, devname: DeviceName) -> bool: + return net.is_netfail_master(devname) + + def is_netfail_primary(self, devname: DeviceName) -> bool: + return net.is_netfail_primary(devname) + + def is_netfail_standby(self, devname: DeviceName) -> bool: + return net.is_netfail_standby(devname) + + def is_physical(self, devname: DeviceName) -> bool: + return os.path.exists(net.sys_dev_path(devname, "device")) + + def settle(self, *, exists=None) -> None: + if exists is not None: + exists = net.sys_dev_path(exists) + util.udevadm_settle(exists=exists) + + def try_set_link_up(self, devname: DeviceName) -> bool: + """Try setting the link to up explicitly and return if it is up. + Not guaranteed to bring the interface up. The caller is expected to + add wait times before retrying.""" + subp.subp(['ip', 'link', 'set', devname, 'up']) + return self.is_up(devname) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/openbsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/openbsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/openbsd.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/openbsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,52 @@ +# Copyright (C) 2019-2020 Gonéri Le Bouder +# +# This file is part of cloud-init. See LICENSE file for license information. + +import os +import platform + +import cloudinit.distros.netbsd +from cloudinit import log as logging +from cloudinit import subp +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class Distro(cloudinit.distros.netbsd.NetBSD): + hostname_conf_fn = '/etc/myname' + + def _read_hostname(self, filename, default=None): + return util.load_file(self.hostname_conf_fn) + + def _write_hostname(self, hostname, filename): + content = hostname + '\n' + util.write_file(self.hostname_conf_fn, content) + + def _get_add_member_to_group_cmd(self, member_name, group_name): + return ['usermod', '-G', group_name, member_name] + + def lock_passwd(self, name): + try: + subp.subp(['usermod', '-p', '*', name]) + except Exception: + util.logexc(LOG, "Failed to lock user %s", name) + raise + + def unlock_passwd(self, name): + pass + + def _get_pkg_cmd_environ(self): + """Return env vars used in OpenBSD package_command operations""" + os_release = platform.release() + os_arch = platform.machine() + e = os.environ.copy() + e['PKG_PATH'] = ( + 'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/' + 'packages/{os_arch}/').format( + os_arch=os_arch, os_release=os_release + ) + return e + + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/opensuse.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/opensuse.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/opensuse.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/opensuse.py 2021-05-11 16:34:29.000000000 +0000 @@ -13,14 +13,12 @@ from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit import helpers -from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util as rhutil from cloudinit.settings import PER_INSTANCE -LOG = logging.getLogger(__name__) - class Distro(distros.Distro): clock_conf_fn = '/etc/sysconfig/clock' @@ -97,7 +95,7 @@ cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def set_timezone(self, tz): tz_file = self._find_tz_file(tz) @@ -129,7 +127,7 @@ if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -144,6 +142,9 @@ return default return hostname + def _get_localhost_ip(self): + return "127.0.1.1" + def _read_hostname_conf(self, filename): conf = HostnameConf(util.load_file(filename)) conf.parse() @@ -160,7 +161,7 @@ if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: conf = None try: @@ -181,7 +182,7 @@ def preferred_ntp_clients(self): """The preferred ntp client is dependent on the version.""" - """Allow distro to determine the preferred ntp client list""" + # Allow distro to determine the preferred ntp client list if not self._preferred_ntp_clients: distro_info = util.system_info()['dist'] name = distro_info[0] diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/parsers/resolv_conf.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/parsers/resolv_conf.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/parsers/resolv_conf.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/parsers/resolv_conf.py 2021-05-11 16:34:29.000000000 +0000 @@ -150,9 +150,10 @@ tail = '' try: (cfg_opt, cfg_values) = head.split(None, 1) - except (IndexError, ValueError): - raise IOError("Incorrectly formatted resolv.conf line %s" - % (i + 1)) + except (IndexError, ValueError) as e: + raise IOError( + "Incorrectly formatted resolv.conf line %s" % (i + 1) + ) from e if cfg_opt not in ['nameserver', 'domain', 'search', 'sortlist', 'options']: raise IOError("Unexpected resolv.conf option %s" % (cfg_opt)) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/rhel.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/rhel.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/rhel.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/rhel.py 2021-05-11 16:34:29.000000000 +0000 @@ -11,6 +11,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util @@ -49,6 +50,10 @@ } } + # Should be fqdn if we can use it + # See: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/deployment_guide/ch-sysconfig # noqa: E501 + prefer_fqdn = True + def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) # This will be used to restrict certain @@ -83,20 +88,13 @@ if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: host_cfg = { 'HOSTNAME': hostname, } rhel_util.update_sysconfig_file(out_fn, host_cfg) - def _select_hostname(self, hostname, fqdn): - # Should be fqdn if we can use it - # See: https://www.centos.org/docs/5/html/Deployment_Guide-en-US/ch-sysconfig.html#s2-sysconfig-network # noqa - if fqdn: - return fqdn - return hostname - def _read_system_hostname(self): if self.uses_systemd(): host_fn = self.systemd_hostname_conf_fn @@ -108,7 +106,7 @@ if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -146,7 +144,7 @@ if pkgs is None: pkgs = [] - if util.which('dnf'): + if subp.which('dnf'): LOG.debug('Using DNF for package management') cmd = ['dnf'] else: @@ -173,7 +171,7 @@ cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/rhel_util.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/rhel_util.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/rhel_util.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/rhel_util.py 2021-05-11 16:34:29.000000000 +0000 @@ -8,7 +8,6 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.distros.parsers.resolv_conf import ResolvConf from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import log as logging @@ -50,29 +49,4 @@ contents = [] return (exists, SysConf(contents)) - -# Helper function to update RHEL/SUSE /etc/resolv.conf -def update_resolve_conf_file(fn, dns_servers, search_servers): - try: - r_conf = ResolvConf(util.load_file(fn)) - r_conf.parse() - except IOError: - util.logexc(LOG, "Failed at parsing %s reverting to an empty " - "instance", fn) - r_conf = ResolvConf('') - r_conf.parse() - if dns_servers: - for s in dns_servers: - try: - r_conf.add_nameserver(s) - except ValueError: - util.logexc(LOG, "Failed at adding nameserver %s", s) - if search_servers: - for s in search_servers: - try: - r_conf.add_search_domain(s) - except ValueError: - util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0o644) - # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/sles.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/sles.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/sles.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/sles.py 2021-05-11 16:34:29.000000000 +0000 @@ -6,10 +6,6 @@ from cloudinit.distros import opensuse -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - class Distro(opensuse.Distro): pass diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/tests/test_init.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/tests/test_init.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/tests/test_init.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/tests/test_init.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,161 @@ +# Copyright (C) 2020 Canonical Ltd. +# +# Author: Daniel Watkins +# +# This file is part of cloud-init. See LICENSE file for license information. +"""Tests for cloudinit/distros/__init__.py""" + +from unittest import mock + +import pytest + +from cloudinit.distros import _get_package_mirror_info, LDH_ASCII_CHARS + +# In newer versions of Python, these characters will be omitted instead +# of substituted because of security concerns. +# See https://bugs.python.org/issue43882 +SECURITY_URL_CHARS = '\n\r\t' + +# Define a set of characters we would expect to be replaced +INVALID_URL_CHARS = [ + chr(x) for x in range(127) + if chr(x) not in LDH_ASCII_CHARS + SECURITY_URL_CHARS +] +for separator in [":", ".", "/", "#", "?", "@", "[", "]"]: + # Remove from the set characters that either separate hostname parts (":", + # "."), terminate hostnames ("/", "#", "?", "@"), or cause Python to be + # unable to parse URLs ("[", "]"). + INVALID_URL_CHARS.remove(separator) + + +class TestGetPackageMirrorInfo: + """ + Tests for cloudinit.distros._get_package_mirror_info. + + These supplement the tests in tests/unittests/test_distros/test_generic.py + which are more focused on testing a single production-like configuration. + These tests are more focused on specific aspects of the unit under test. + """ + + @pytest.mark.parametrize('mirror_info,expected', [ + # Empty info gives empty return + ({}, {}), + # failsafe values used if present + ({'failsafe': {'primary': 'http://value', 'security': 'http://other'}}, + {'primary': 'http://value', 'security': 'http://other'}), + # search values used if present + ({'search': {'primary': ['http://value'], + 'security': ['http://other']}}, + {'primary': ['http://value'], 'security': ['http://other']}), + # failsafe values used if search value not present + ({'search': {'primary': ['http://value']}, + 'failsafe': {'security': 'http://other'}}, + {'primary': ['http://value'], 'security': 'http://other'}) + ]) + def test_get_package_mirror_info_failsafe(self, mirror_info, expected): + """ + Test the interaction between search and failsafe inputs + + (This doesn't test the case where the mirror_filter removes all search + options; test_failsafe_used_if_all_search_results_filtered_out covers + that.) + """ + assert expected == _get_package_mirror_info(mirror_info, + mirror_filter=lambda x: x) + + def test_failsafe_used_if_all_search_results_filtered_out(self): + """Test the failsafe option used if all search options eliminated.""" + mirror_info = { + 'search': {'primary': ['http://value']}, + 'failsafe': {'primary': 'http://other'} + } + assert {'primary': 'http://other'} == _get_package_mirror_info( + mirror_info, mirror_filter=lambda x: False) + + @pytest.mark.parametrize('allow_ec2_mirror, platform_type', [ + (True, 'ec2') + ]) + @pytest.mark.parametrize('availability_zone,region,patterns,expected', ( + # Test ec2_region alone + ('fk-fake-1f', None, ['http://EC2-%(ec2_region)s/ubuntu'], + ['http://ec2-fk-fake-1/ubuntu']), + # Test availability_zone alone + ('fk-fake-1f', None, ['http://AZ-%(availability_zone)s/ubuntu'], + ['http://az-fk-fake-1f/ubuntu']), + # Test region alone + (None, 'fk-fake-1', ['http://RG-%(region)s/ubuntu'], + ['http://rg-fk-fake-1/ubuntu']), + # Test that ec2_region is not available for non-matching AZs + ('fake-fake-1f', None, + ['http://EC2-%(ec2_region)s/ubuntu', + 'http://AZ-%(availability_zone)s/ubuntu'], + ['http://az-fake-fake-1f/ubuntu']), + # Test that template order maintained + (None, 'fake-region', + ['http://RG-%(region)s-2/ubuntu', 'http://RG-%(region)s-1/ubuntu'], + ['http://rg-fake-region-2/ubuntu', 'http://rg-fake-region-1/ubuntu']), + # Test that non-ASCII hostnames are IDNA encoded; + # "IDNA-ТεЅТ̣".encode('idna') == b"xn--idna--4kd53hh6aba3q" + (None, 'ТεЅТ̣', ['http://www.IDNA-%(region)s.com/ubuntu'], + ['http://www.xn--idna--4kd53hh6aba3q.com/ubuntu']), + # Test that non-ASCII hostnames with a port are IDNA encoded; + # "IDNA-ТεЅТ̣".encode('idna') == b"xn--idna--4kd53hh6aba3q" + (None, 'ТεЅТ̣', ['http://www.IDNA-%(region)s.com:8080/ubuntu'], + ['http://www.xn--idna--4kd53hh6aba3q.com:8080/ubuntu']), + # Test that non-ASCII non-hostname parts of URLs are unchanged + (None, 'ТεЅТ̣', ['http://www.example.com/%(region)s/ubuntu'], + ['http://www.example.com/ТεЅТ̣/ubuntu']), + # Test that IPv4 addresses are unchanged + (None, 'fk-fake-1', ['http://192.168.1.1:8080/%(region)s/ubuntu'], + ['http://192.168.1.1:8080/fk-fake-1/ubuntu']), + # Test that IPv6 addresses are unchanged + (None, 'fk-fake-1', + ['http://[2001:67c:1360:8001::23]/%(region)s/ubuntu'], + ['http://[2001:67c:1360:8001::23]/fk-fake-1/ubuntu']), + # Test that unparseable URLs are filtered out of the mirror list + (None, 'inv[lid', + ['http://%(region)s.in.hostname/should/be/filtered', + 'http://but.not.in.the.path/%(region)s'], + ['http://but.not.in.the.path/inv[lid']), + (None, '-some-region-', + ['http://-lead-ing.%(region)s.trail-ing-.example.com/ubuntu'], + ['http://lead-ing.some-region.trail-ing.example.com/ubuntu']), + ) + tuple( + # Dynamically generate a test case for each non-LDH + # (Letters/Digits/Hyphen) ASCII character, testing that it is + # substituted with a hyphen + (None, 'fk{0}fake{0}1'.format(invalid_char), + ['http://%(region)s/ubuntu'], ['http://fk-fake-1/ubuntu']) + for invalid_char in INVALID_URL_CHARS + )) + def test_valid_substitution(self, + allow_ec2_mirror, + platform_type, + availability_zone, + region, + patterns, + expected): + """Test substitution works as expected.""" + flag_path = "cloudinit.distros." \ + "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES" + + m_data_source = mock.Mock( + availability_zone=availability_zone, + region=region, + platform_type=platform_type + ) + mirror_info = {'search': {'primary': patterns}} + + with mock.patch(flag_path, allow_ec2_mirror): + ret = _get_package_mirror_info( + mirror_info, + data_source=m_data_source, + mirror_filter=lambda x: x + ) + print(allow_ec2_mirror) + print(platform_type) + print(availability_zone) + print(region) + print(patterns) + print(expected) + assert {'primary': expected} == ret diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/tests/test_networking.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/tests/test_networking.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/tests/test_networking.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/tests/test_networking.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,223 @@ +from unittest import mock + +import pytest + +from cloudinit import net +from cloudinit.distros.networking import ( + BSDNetworking, + LinuxNetworking, + Networking, +) + +# See https://docs.pytest.org/en/stable/example +# /parametrize.html#parametrizing-conditional-raising +from contextlib import ExitStack as does_not_raise + + +@pytest.yield_fixture +def generic_networking_cls(): + """Returns a direct Networking subclass which errors on /sys usage. + + This enables the direct testing of functionality only present on the + ``Networking`` super-class, and provides a check on accidentally using /sys + in that context. + """ + + class TestNetworking(Networking): + def is_physical(self, *args, **kwargs): + raise NotImplementedError + + def settle(self, *args, **kwargs): + raise NotImplementedError + + def try_set_link_up(self, *args, **kwargs): + raise NotImplementedError + + error = AssertionError("Unexpectedly used /sys in generic networking code") + with mock.patch( + "cloudinit.net.get_sys_class_path", side_effect=error, + ): + yield TestNetworking + + +@pytest.yield_fixture +def sys_class_net(tmpdir): + sys_class_net_path = tmpdir.join("sys/class/net") + sys_class_net_path.ensure_dir() + with mock.patch( + "cloudinit.net.get_sys_class_path", + return_value=sys_class_net_path.strpath + "/", + ): + yield sys_class_net_path + + +class TestBSDNetworkingIsPhysical: + def test_raises_notimplementederror(self): + with pytest.raises(NotImplementedError): + BSDNetworking().is_physical("eth0") + + +class TestLinuxNetworkingIsPhysical: + def test_returns_false_by_default(self, sys_class_net): + assert not LinuxNetworking().is_physical("eth0") + + def test_returns_false_if_devname_exists_but_not_physical( + self, sys_class_net + ): + devname = "eth0" + sys_class_net.join(devname).mkdir() + assert not LinuxNetworking().is_physical(devname) + + def test_returns_true_if_device_is_physical(self, sys_class_net): + devname = "eth0" + device_dir = sys_class_net.join(devname) + device_dir.mkdir() + device_dir.join("device").write("") + + assert LinuxNetworking().is_physical(devname) + + +class TestBSDNetworkingTrySetLinkUp: + def test_raises_notimplementederror(self): + with pytest.raises(NotImplementedError): + BSDNetworking().try_set_link_up("eth0") + + +@mock.patch("cloudinit.net.is_up") +@mock.patch("cloudinit.distros.networking.subp.subp") +class TestLinuxNetworkingTrySetLinkUp: + def test_calls_subp_return_true(self, m_subp, m_is_up): + devname = "eth0" + m_is_up.return_value = True + is_success = LinuxNetworking().try_set_link_up(devname) + + assert (mock.call(['ip', 'link', 'set', devname, 'up']) == + m_subp.call_args_list[-1]) + assert is_success + + def test_calls_subp_return_false(self, m_subp, m_is_up): + devname = "eth0" + m_is_up.return_value = False + is_success = LinuxNetworking().try_set_link_up(devname) + + assert (mock.call(['ip', 'link', 'set', devname, 'up']) == + m_subp.call_args_list[-1]) + assert not is_success + + +class TestBSDNetworkingSettle: + def test_settle_doesnt_error(self): + # This also implicitly tests that it doesn't use subp.subp + BSDNetworking().settle() + + +@pytest.mark.usefixtures("sys_class_net") +@mock.patch("cloudinit.distros.networking.util.udevadm_settle", autospec=True) +class TestLinuxNetworkingSettle: + def test_no_arguments(self, m_udevadm_settle): + LinuxNetworking().settle() + + assert [mock.call(exists=None)] == m_udevadm_settle.call_args_list + + def test_exists_argument(self, m_udevadm_settle): + LinuxNetworking().settle(exists="ens3") + + expected_path = net.sys_dev_path("ens3") + assert [ + mock.call(exists=expected_path) + ] == m_udevadm_settle.call_args_list + + +class TestNetworkingWaitForPhysDevs: + @pytest.fixture + def wait_for_physdevs_netcfg(self): + """This config is shared across all the tests in this class.""" + + def ethernet(mac, name, driver=None, device_id=None): + v2_cfg = {"set-name": name, "match": {"macaddress": mac}} + if driver: + v2_cfg["match"].update({"driver": driver}) + if device_id: + v2_cfg["match"].update({"device_id": device_id}) + + return v2_cfg + + physdevs = [ + ["aa:bb:cc:dd:ee:ff", "eth0", "virtio", "0x1000"], + ["00:11:22:33:44:55", "ens3", "e1000", "0x1643"], + ] + netcfg = { + "version": 2, + "ethernets": {args[1]: ethernet(*args) for args in physdevs}, + } + return netcfg + + def test_skips_settle_if_all_present( + self, generic_networking_cls, wait_for_physdevs_netcfg, + ): + networking = generic_networking_cls() + with mock.patch.object( + networking, "get_interfaces_by_mac" + ) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.side_effect = iter( + [{"aa:bb:cc:dd:ee:ff": "eth0", "00:11:22:33:44:55": "ens3"}] + ) + with mock.patch.object( + networking, "settle", autospec=True + ) as m_settle: + networking.wait_for_physdevs(wait_for_physdevs_netcfg) + assert 0 == m_settle.call_count + + def test_calls_udev_settle_on_missing( + self, generic_networking_cls, wait_for_physdevs_netcfg, + ): + networking = generic_networking_cls() + with mock.patch.object( + networking, "get_interfaces_by_mac" + ) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.side_effect = iter( + [ + { + "aa:bb:cc:dd:ee:ff": "eth0" + }, # first call ens3 is missing + { + "aa:bb:cc:dd:ee:ff": "eth0", + "00:11:22:33:44:55": "ens3", + }, # second call has both + ] + ) + with mock.patch.object( + networking, "settle", autospec=True + ) as m_settle: + networking.wait_for_physdevs(wait_for_physdevs_netcfg) + m_settle.assert_called_with(exists="ens3") + + @pytest.mark.parametrize( + "strict,expectation", + [(True, pytest.raises(RuntimeError)), (False, does_not_raise())], + ) + def test_retrying_and_strict_behaviour( + self, + strict, + expectation, + generic_networking_cls, + wait_for_physdevs_netcfg, + ): + networking = generic_networking_cls() + with mock.patch.object( + networking, "get_interfaces_by_mac" + ) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.return_value = {} + + with mock.patch.object( + networking, "settle", autospec=True + ) as m_settle: + with expectation: + networking.wait_for_physdevs( + wait_for_physdevs_netcfg, strict=strict + ) + + assert ( + 5 * len(wait_for_physdevs_netcfg["ethernets"]) + == m_settle.call_count + ) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/distros/ubuntu.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/ubuntu.py --- cloud-init-20.1-10-g71af48df/cloudinit/distros/ubuntu.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/distros/ubuntu.py 2021-05-11 16:34:29.000000000 +0000 @@ -11,13 +11,10 @@ from cloudinit.distros import debian from cloudinit.distros import PREFERRED_NTP_CLIENTS -from cloudinit import log as logging from cloudinit import util import copy -LOG = logging.getLogger(__name__) - class Distro(debian.Distro): @@ -49,7 +46,5 @@ copy.deepcopy(PREFERRED_NTP_CLIENTS)) return self._preferred_ntp_clients - pass - # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/dmi.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/dmi.py --- cloud-init-20.1-10-g71af48df/cloudinit/dmi.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/dmi.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,163 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from cloudinit import log as logging +from cloudinit import subp +from cloudinit.util import is_container, is_FreeBSD + +from collections import namedtuple +import os + +LOG = logging.getLogger(__name__) + +# Path for DMI Data +DMI_SYS_PATH = "/sys/class/dmi/id" + +kdmi = namedtuple('KernelNames', ['linux', 'freebsd']) +kdmi.__new__.defaults__ = (None, None) + +# FreeBSD's kenv(1) and Linux /sys/class/dmi/id/* both use different names from +# dmidecode. The values are the same, and ultimately what we're interested in. +# These tools offer a "cheaper" way to access those values over dmidecode. +# This is our canonical translation table. If we add more tools on other +# platforms to find dmidecode's values, their keys need to be put in here. +DMIDECODE_TO_KERNEL = { + 'baseboard-asset-tag': kdmi('board_asset_tag', 'smbios.planar.tag'), + 'baseboard-manufacturer': kdmi('board_vendor', 'smbios.planar.maker'), + 'baseboard-product-name': kdmi('board_name', 'smbios.planar.product'), + 'baseboard-serial-number': kdmi('board_serial', 'smbios.planar.serial'), + 'baseboard-version': kdmi('board_version', 'smbios.planar.version'), + 'bios-release-date': kdmi('bios_date', 'smbios.bios.reldate'), + 'bios-vendor': kdmi('bios_vendor', 'smbios.bios.vendor'), + 'bios-version': kdmi('bios_version', 'smbios.bios.version'), + 'chassis-asset-tag': kdmi('chassis_asset_tag', 'smbios.chassis.tag'), + 'chassis-manufacturer': kdmi('chassis_vendor', 'smbios.chassis.maker'), + 'chassis-serial-number': kdmi('chassis_serial', 'smbios.chassis.serial'), + 'chassis-version': kdmi('chassis_version', 'smbios.chassis.version'), + 'system-manufacturer': kdmi('sys_vendor', 'smbios.system.maker'), + 'system-product-name': kdmi('product_name', 'smbios.system.product'), + 'system-serial-number': kdmi('product_serial', 'smbios.system.serial'), + 'system-uuid': kdmi('product_uuid', 'smbios.system.uuid'), + 'system-version': kdmi('product_version', 'smbios.system.version'), +} + + +def _read_dmi_syspath(key): + """ + Reads dmi data from /sys/class/dmi/id + """ + kmap = DMIDECODE_TO_KERNEL.get(key) + if kmap is None or kmap.linux is None: + return None + dmi_key_path = "{0}/{1}".format(DMI_SYS_PATH, kmap.linux) + LOG.debug("querying dmi data %s", dmi_key_path) + if not os.path.exists(dmi_key_path): + LOG.debug("did not find %s", dmi_key_path) + return None + + try: + with open(dmi_key_path, "rb") as fp: + key_data = fp.read() + except PermissionError: + LOG.debug("Could not read %s", dmi_key_path) + return None + + # uninitialized dmi values show as all \xff and /sys appends a '\n'. + # in that event, return empty string. + if key_data == b'\xff' * (len(key_data) - 1) + b'\n': + key_data = b"" + + try: + return key_data.decode('utf8').strip() + except UnicodeDecodeError as e: + LOG.error("utf-8 decode of content (%s) in %s failed: %s", + dmi_key_path, key_data, e) + + return None + + +def _read_kenv(key): + """ + Reads dmi data from FreeBSD's kenv(1) + """ + kmap = DMIDECODE_TO_KERNEL.get(key) + if kmap is None or kmap.freebsd is None: + return None + + LOG.debug("querying dmi data %s", kmap.freebsd) + + try: + cmd = ["kenv", "-q", kmap.freebsd] + (result, _err) = subp.subp(cmd) + result = result.strip() + LOG.debug("kenv returned '%s' for '%s'", result, kmap.freebsd) + return result + except subp.ProcessExecutionError as e: + LOG.debug('failed kenv cmd: %s\n%s', cmd, e) + return None + + return None + + +def _call_dmidecode(key, dmidecode_path): + """ + Calls out to dmidecode to get the data out. This is mostly for supporting + OS's without /sys/class/dmi/id support. + """ + try: + cmd = [dmidecode_path, "--string", key] + (result, _err) = subp.subp(cmd) + result = result.strip() + LOG.debug("dmidecode returned '%s' for '%s'", result, key) + if result.replace(".", "") == "": + return "" + return result + except subp.ProcessExecutionError as e: + LOG.debug('failed dmidecode cmd: %s\n%s', cmd, e) + return None + + +def read_dmi_data(key): + """ + Wrapper for reading DMI data. + + If running in a container return None. This is because DMI data is + assumed to be not useful in a container as it does not represent the + container but rather the host. + + This will do the following (returning the first that produces a + result): + 1) Use a mapping to translate `key` from dmidecode naming to + sysfs naming and look in /sys/class/dmi/... for a value. + 2) Use `key` as a sysfs key directly and look in /sys/class/dmi/... + 3) Fall-back to passing `key` to `dmidecode --string`. + + If all of the above fail to find a value, None will be returned. + """ + + if is_container(): + return None + + if is_FreeBSD(): + return _read_kenv(key) + + syspath_value = _read_dmi_syspath(key) + if syspath_value is not None: + return syspath_value + + def is_x86(arch): + return (arch == 'x86_64' or (arch[0] == 'i' and arch[2:] == '86')) + + # running dmidecode can be problematic on some arches (LP: #1243287) + uname_arch = os.uname()[4] + if not (is_x86(uname_arch) or uname_arch in ('aarch64', 'amd64')): + LOG.debug("dmidata is not supported on %s", uname_arch) + return None + + dmidecode_path = subp.which('dmidecode') + if dmidecode_path: + return _call_dmidecode(key, dmidecode_path) + + LOG.warning("did not find either path %s or dmidecode command", + DMI_SYS_PATH) + return None + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/features.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/features.py --- cloud-init-20.1-10-g71af48df/cloudinit/features.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/features.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,56 @@ +# This file is part of cloud-init. See LICENSE file for license information. +""" +Feature flags are used as a way to easily toggle configuration +**at build time**. They are provided to accommodate feature deprecation and +downstream configuration changes. + +Currently used upstream values for feature flags are set in +``cloudinit/features.py``. Overrides to these values (typically via quilt +patch) can be placed +in a file called ``feature_overrides.py`` in the same directory. Any value +set in ``feature_overrides.py`` will override the original value set +in ``features.py``. + +Each flag should include a short comment regarding the reason for +the flag and intended lifetime. + +Tests are required for new feature flags, and tests must verify +all valid states of a flag, not just the default state. +""" + +ERROR_ON_USER_DATA_FAILURE = True +""" +If there is a failure in obtaining user data (i.e., #include or +decompress fails) and ``ERROR_ON_USER_DATA_FAILURE`` is ``False``, +cloud-init will log a warning and proceed. If it is ``True``, +cloud-init will instead raise an exception. + +As of 20.3, ``ERROR_ON_USER_DATA_FAILURE`` is ``True``. + +(This flag can be removed after Focal is no longer supported.) +""" + + +ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES = False +""" +When configuring apt mirrors, if +``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``True`` cloud-init +will detect that a datasource's ``availability_zone`` property looks +like an EC2 availability zone and set the ``ec2_region`` variable when +generating mirror URLs; this can lead to incorrect mirrors being +configured in clouds whose AZs follow EC2's naming pattern. + +As of 20.3, ``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``False`` +so we no longer include ``ec2_region`` in mirror determination on +non-AWS cloud platforms. + +If the old behavior is desired, users can provide the appropriate +mirrors via :py:mod:`apt: ` +directives in cloud-config. +""" + +try: + # pylint: disable=wildcard-import + from cloudinit.feature_overrides import * # noqa +except ImportError: + pass diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/gpg.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/gpg.py --- cloud-init-20.1-10-g71af48df/cloudinit/gpg.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/gpg.py 2021-05-11 16:34:29.000000000 +0000 @@ -8,7 +8,7 @@ """gpg.py - Collection of gpg key related functions""" from cloudinit import log as logging -from cloudinit import util +from cloudinit import subp import time @@ -18,9 +18,9 @@ def export_armour(key): """Export gpg key, armoured key gets returned""" try: - (armour, _) = util.subp(["gpg", "--export", "--armour", key], + (armour, _) = subp.subp(["gpg", "--export", "--armour", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: # debug, since it happens for any key not on the system initially LOG.debug('Failed to export armoured key "%s": %s', key, error) armour = None @@ -42,7 +42,7 @@ @param retries: an iterable of sleep lengths for retries. Use None to indicate no retries.""" LOG.debug("Importing key '%s' from keyserver '%s'", key, keyserver) - cmd = ["gpg", "--keyserver=%s" % keyserver, "--recv-keys", key] + cmd = ["gpg", "--no-tty", "--keyserver=%s" % keyserver, "--recv-keys", key] if retries is None: retries = [] trynum = 0 @@ -51,11 +51,11 @@ while True: trynum += 1 try: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) LOG.debug("Imported key '%s' from keyserver '%s' on try %d", key, keyserver, trynum) return - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: error = e try: naplen = next(sleeps) @@ -63,18 +63,19 @@ "Import failed with exit code %d, will try again in %ss", error.exit_code, naplen) time.sleep(naplen) - except StopIteration: + except StopIteration as e: raise ValueError( ("Failed to import key '%s' from keyserver '%s' " - "after %d tries: %s") % (key, keyserver, trynum, error)) + "after %d tries: %s") % (key, keyserver, trynum, error) + ) from e def delete_key(key): """Delete the specified key from the local gpg ring""" try: - util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + subp.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: LOG.warning('Failed delete key "%s": %s', key, error) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/handlers/boot_hook.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/boot_hook.py --- cloud-init-20.1-10-g71af48df/cloudinit/handlers/boot_hook.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/boot_hook.py 2021-05-11 16:34:29.000000000 +0000 @@ -12,6 +12,7 @@ from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_ALWAYS) @@ -48,8 +49,8 @@ env = os.environ.copy() if self.instance_id is not None: env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError: + subp.subp([filepath], env=env) + except subp.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) except Exception: util.logexc(LOG, "Boothooks unknown error when running %s", diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/handlers/jinja_template.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/jinja_template.py --- cloud-init-20.1-10-g71af48df/cloudinit/handlers/jinja_template.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/jinja_template.py 2021-05-11 16:34:29.000000000 +0000 @@ -83,7 +83,8 @@ if e.errno == EACCES: raise RuntimeError( 'Cannot render jinja template vars. No read permission on' - " '%s'. Try sudo" % instance_data_file) + " '%s'. Try sudo" % instance_data_file + ) from e rendered_payload = render_jinja_payload( payload, payload_fn, instance_data, debug) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/handlers/shell_script.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/shell_script.py --- cloud-init-20.1-10-g71af48df/cloudinit/handlers/shell_script.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/shell_script.py 2021-05-11 16:34:29.000000000 +0000 @@ -11,13 +11,10 @@ import os from cloudinit import handlers -from cloudinit import log as logging from cloudinit import util from cloudinit.settings import (PER_ALWAYS) -LOG = logging.getLogger(__name__) - class ShellScriptPartHandler(handlers.Handler): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/handlers/upstart_job.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/upstart_job.py --- cloud-init-20.1-10-g71af48df/cloudinit/handlers/upstart_job.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/handlers/upstart_job.py 2021-05-11 16:34:29.000000000 +0000 @@ -13,6 +13,7 @@ from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_INSTANCE) @@ -52,7 +53,7 @@ util.write_file(path, payload, 0o644) if SUITABLE_UPSTART: - util.subp(["initctl", "reload-configuration"], capture=False) + subp.subp(["initctl", "reload-configuration"], capture=False) def _has_suitable_upstart(): @@ -63,7 +64,7 @@ if not os.path.exists("/sbin/initctl"): return False try: - (version_out, _err) = util.subp(["initctl", "version"]) + (version_out, _err) = subp.subp(["initctl", "version"]) except Exception: util.logexc(LOG, "initctl version failed") return False @@ -77,7 +78,7 @@ if not os.path.exists("/usr/bin/dpkg-query"): return False try: - (dpkg_ver, _err) = util.subp(["dpkg-query", + (dpkg_ver, _err) = subp.subp(["dpkg-query", "--showformat=${Version}", "--show", "upstart"], rcs=[0, 1]) except Exception: @@ -86,9 +87,9 @@ try: good = "1.8-0ubuntu1.2" - util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) + subp.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code == 1: pass else: diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/helpers.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/helpers.py --- cloud-init-20.1-10-g71af48df/cloudinit/helpers.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/helpers.py 2021-05-11 16:34:29.000000000 +0000 @@ -20,6 +20,7 @@ from cloudinit import log as logging from cloudinit import type_utils +from cloudinit import persistence from cloudinit import util LOG = logging.getLogger(__name__) @@ -230,6 +231,10 @@ cc_paths = ['cloud_config'] if self._include_vendor: + # the order is important here: we want vendor2 + # (dynamic vendor data from OpenStack) + # to override vendor (static data from OpenStack) + cc_paths.append('vendor2_cloud_config') cc_paths.append('vendor_cloud_config') for cc_p in cc_paths: @@ -313,7 +318,9 @@ return list(self.registered.items()) -class Paths(object): +class Paths(persistence.CloudInitPickleMixin): + _ci_pkl_version = 1 + def __init__(self, path_cfgs, ds=None): self.cfgs = path_cfgs # Populate all the initial paths @@ -337,9 +344,12 @@ "obj_pkl": "obj.pkl", "cloud_config": "cloud-config.txt", "vendor_cloud_config": "vendor-cloud-config.txt", + "vendor2_cloud_config": "vendor2-cloud-config.txt", "data": "data", "vendordata_raw": "vendor-data.txt", + "vendordata2_raw": "vendor-data2.txt", "vendordata": "vendor-data.txt.i", + "vendordata2": "vendor-data2.txt.i", "instance_id": ".instance-id", "manual_clean_marker": "manual-clean", "warnings": "warnings", @@ -347,6 +357,18 @@ # Set when a datasource becomes active self.datasource = ds + def _unpickle(self, ci_pkl_version: int) -> None: + """Perform deserialization fixes for Paths.""" + if not hasattr(self, "run_dir"): + # On older versions of cloud-init the Paths class do not + # have the run_dir attribute. This is problematic because + # when loading the pickle object on newer versions of cloud-init + # we will rely on this attribute. To fix that, we are now + # manually adding that attribute here. + self.run_dir = Paths( + path_cfgs=self.cfgs, + ds=self.datasource).run_dir + # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): return self._get_path(self.instance_link, name) @@ -451,8 +473,4 @@ contents = '\n'.join([header, contents, '']) return contents - -def identity(object): - return object - # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/log.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/log.py --- cloud-init-20.1-10-g71af48df/cloudinit/log.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/log.py 2021-05-11 16:34:29.000000000 +0000 @@ -122,17 +122,12 @@ return logging.getLogger(name) -# Fixes this annoyance... -# No handlers could be found for logger XXX annoying output... -try: - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass - - def _resetLogger(log): + """Remove all current handlers, unset log level and add a NullHandler. + + (Adding the NullHandler avoids "No handlers could be found for logger XXX" + messages.) + """ if not log: return handlers = list(log.handlers) @@ -141,7 +136,7 @@ h.close() log.removeHandler(h) log.setLevel(NOTSET) - log.addHandler(NullHandler()) + log.addHandler(logging.NullHandler()) def resetLogging(): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/mergers/__init__.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/mergers/__init__.py --- cloud-init-20.1-10-g71af48df/cloudinit/mergers/__init__.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/mergers/__init__.py 2021-05-11 16:34:29.000000000 +0000 @@ -7,12 +7,10 @@ import re from cloudinit import importer -from cloudinit import log as logging from cloudinit import type_utils NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") -LOG = logging.getLogger(__name__) DEF_MERGE_TYPE = "list()+dict()+str()" MERGER_PREFIX = 'm_' MERGER_ATTR = 'Merger' diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/bsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/bsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/bsd.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/bsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,167 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re + +from cloudinit import log as logging +from cloudinit import net +from cloudinit import util +from cloudinit import subp +from cloudinit.distros.parsers.resolv_conf import ResolvConf +from cloudinit.distros import bsd_utils + +from . import renderer + +LOG = logging.getLogger(__name__) + + +class BSDRenderer(renderer.Renderer): + resolv_conf_fn = 'etc/resolv.conf' + rc_conf_fn = 'etc/rc.conf' + + def get_rc_config_value(self, key): + fn = subp.target_path(self.target, self.rc_conf_fn) + bsd_utils.get_rc_config_value(key, fn=fn) + + def set_rc_config_value(self, key, value): + fn = subp.target_path(self.target, self.rc_conf_fn) + bsd_utils.set_rc_config_value(key, value, fn=fn) + + def __init__(self, config=None): + if not config: + config = {} + self.target = None + self.interface_configurations = {} + self._postcmds = config.get('postcmds', True) + + def _ifconfig_entries(self, settings, target=None): + ifname_by_mac = net.get_interfaces_by_mac() + for interface in settings.iter_interfaces(): + device_name = interface.get("name") + device_mac = interface.get("mac_address") + if device_name and re.match(r'^lo\d+$', device_name): + continue + if device_mac not in ifname_by_mac: + LOG.info('Cannot find any device with MAC %s', device_mac) + elif device_mac and device_name: + cur_name = ifname_by_mac[device_mac] + if cur_name != device_name: + LOG.info('netif service will rename interface %s to %s', + cur_name, device_name) + try: + self.rename_interface(cur_name, device_name) + except NotImplementedError: + LOG.error(( + 'Interface renaming is ' + 'not supported on this OS')) + device_name = cur_name + + else: + device_name = ifname_by_mac[device_mac] + + LOG.info('Configuring interface %s', device_name) + + self.interface_configurations[device_name] = 'DHCP' + + for subnet in interface.get("subnets", []): + if subnet.get('type') == 'static': + if not subnet.get('netmask'): + LOG.debug( + 'Skipping IP %s, because there is no netmask', + subnet.get('address') + ) + continue + LOG.debug('Configuring dev %s with %s / %s', device_name, + subnet.get('address'), subnet.get('netmask')) + + self.interface_configurations[device_name] = { + 'address': subnet.get('address'), + 'netmask': subnet.get('netmask'), + } + + def _route_entries(self, settings, target=None): + routes = list(settings.iter_routes()) + for interface in settings.iter_interfaces(): + subnets = interface.get("subnets", []) + for subnet in subnets: + if subnet.get('type') != 'static': + continue + gateway = subnet.get('gateway') + if gateway and len(gateway.split('.')) == 4: + routes.append({ + 'network': '0.0.0.0', + 'netmask': '0.0.0.0', + 'gateway': gateway}) + routes += subnet.get('routes', []) + for route in routes: + network = route.get('network') + if not network: + LOG.debug('Skipping a bad route entry') + continue + netmask = route.get('netmask') + gateway = route.get('gateway') + self.set_route(network, netmask, gateway) + + def _resolve_conf(self, settings, target=None): + nameservers = settings.dns_nameservers + searchdomains = settings.dns_searchdomains + for interface in settings.iter_interfaces(): + for subnet in interface.get("subnets", []): + if 'dns_nameservers' in subnet: + nameservers.extend(subnet['dns_nameservers']) + if 'dns_search' in subnet: + searchdomains.extend(subnet['dns_search']) + # Try to read the /etc/resolv.conf or just start from scratch if that + # fails. + try: + resolvconf = ResolvConf(util.load_file(subp.target_path( + target, self.resolv_conf_fn))) + resolvconf.parse() + except IOError: + util.logexc(LOG, "Failed to parse %s, use new empty file", + subp.target_path(target, self.resolv_conf_fn)) + resolvconf = ResolvConf('') + resolvconf.parse() + + # Add some nameservers + for server in nameservers: + try: + resolvconf.add_nameserver(server) + except ValueError: + util.logexc(LOG, "Failed to add nameserver %s", server) + + # And add any searchdomains. + for domain in searchdomains: + try: + resolvconf.add_search_domain(domain) + except ValueError: + util.logexc(LOG, "Failed to add search domain %s", domain) + util.write_file( + subp.target_path(target, self.resolv_conf_fn), + str(resolvconf), 0o644) + + def render_network_state(self, network_state, templates=None, target=None): + self._ifconfig_entries(settings=network_state) + self._route_entries(settings=network_state) + self._resolve_conf(settings=network_state) + + self.write_config() + self.start_services(run=self._postcmds) + + def dhcp_interfaces(self): + ic = self.interface_configurations.items + return [k for k, v in ic() if v == 'DHCP'] + + def start_services(self, run=False): + raise NotImplementedError() + + def write_config(self, target=None): + raise NotImplementedError() + + def set_gateway(self, gateway): + raise NotImplementedError() + + def rename_interface(self, cur_name, device_name): + raise NotImplementedError() + + def set_route(self, network, netmask, gateway): + raise NotImplementedError() diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/cmdline.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/cmdline.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/cmdline.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/cmdline.py 2021-05-11 16:34:29.000000000 +0000 @@ -12,6 +12,7 @@ import io import logging import os +import shlex from cloudinit import util @@ -27,16 +28,12 @@ """ABC for net config sources that read config written by initramfses""" @abc.abstractmethod - def is_applicable(self): - # type: () -> bool + def is_applicable(self) -> bool: """Is this initramfs config source applicable to the current system?""" - pass @abc.abstractmethod - def render_config(self): - # type: () -> dict + def render_config(self) -> dict: """Render a v1 network config from the initramfs configuration""" - pass class KlibcNetworkConfigSource(InitramfsNetworkConfigSource): @@ -65,8 +62,7 @@ if mac_addr: self._mac_addrs[k] = mac_addr - def is_applicable(self): - # type: () -> bool + def is_applicable(self) -> bool: """ Return whether this system has klibc initramfs network config or not @@ -77,15 +73,15 @@ (ii) an open-iscsi interface file is present in the system """ if self._files: - if 'ip=' in self._cmdline or 'ip6=' in self._cmdline: - return True + for item in shlex.split(self._cmdline): + if item.startswith('ip=') or item.startswith('ip6='): + return True if os.path.exists(_OPEN_ISCSI_INTERFACE_FILE): # iBft can configure networking without ip= return True return False - def render_config(self): - # type: () -> dict + def render_config(self) -> dict: return config_from_klibc_net_cfg( files=self._files, mac_addrs=self._mac_addrs, ) @@ -118,8 +114,8 @@ data = util.load_shell_content(content) try: name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6'] - except KeyError: - raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") + except KeyError as e: + raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") from e # ipconfig on precise does not write PROTO # IPv6 config gives us IPV6PROTO, not PROTO. diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/dhcp.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/dhcp.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/dhcp.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/dhcp.py 2021-05-11 16:34:29.000000000 +0000 @@ -17,6 +17,7 @@ has_url_connectivity) from cloudinit.net.network_state import mask_and_ipv4_to_bcast_addr as bcip from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -30,19 +31,18 @@ Current uses are DataSourceAzure and DataSourceEc2 during ephemeral boot to scrape metadata. """ - pass class NoDHCPLeaseError(Exception): """Raised when unable to get a DHCP lease.""" - pass class EphemeralDHCPv4(object): - def __init__(self, iface=None, connectivity_url=None): + def __init__(self, iface=None, connectivity_url=None, dhcp_log_func=None): self.iface = iface self._ephipv4 = None self.lease = None + self.dhcp_log_func = dhcp_log_func self.connectivity_url = connectivity_url def __enter__(self): @@ -80,9 +80,10 @@ if self.lease: return self.lease try: - leases = maybe_perform_dhcp_discovery(self.iface) - except InvalidDHCPLeaseFileError: - raise NoDHCPLeaseError() + leases = maybe_perform_dhcp_discovery( + self.iface, self.dhcp_log_func) + except InvalidDHCPLeaseFileError as e: + raise NoDHCPLeaseError() from e if not leases: raise NoDHCPLeaseError() self.lease = leases[-1] @@ -130,13 +131,15 @@ result[internal_mapping] = self.lease.get(different_names) -def maybe_perform_dhcp_discovery(nic=None): +def maybe_perform_dhcp_discovery(nic=None, dhcp_log_func=None): """Perform dhcp discovery if nic valid and dhclient command exists. If the nic is invalid or undiscoverable or dhclient command is not found, skip dhcp_discovery and return an empty dict. @param nic: Name of the network interface we want to run dhclient on. + @param dhcp_log_func: A callable accepting the dhclient output and error + streams. @return: A list of dicts representing dhcp options for each lease obtained from the dhclient discovery if run, otherwise an empty list is returned. @@ -150,7 +153,7 @@ LOG.debug( 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) return [] - dhclient_path = util.which('dhclient') + dhclient_path = subp.which('dhclient') if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return [] @@ -158,7 +161,7 @@ prefix='cloud-init-dhcp-', needs_exe=True) as tdir: # Use /var/tmp because /run/cloud-init/tmp is mounted noexec - return dhcp_discovery(dhclient_path, nic, tdir) + return dhcp_discovery(dhclient_path, nic, tdir, dhcp_log_func) def parse_dhcp_lease_file(lease_file): @@ -192,13 +195,15 @@ return dhcp_leases -def dhcp_discovery(dhclient_cmd_path, interface, cleandir): +def dhcp_discovery(dhclient_cmd_path, interface, cleandir, dhcp_log_func=None): """Run dhclient on the interface without scripts or filesystem artifacts. @param dhclient_cmd_path: Full path to the dhclient used. @param interface: Name of the network inteface on which to dhclient. @param cleandir: The directory from which to run dhclient as well as store dhcp leases. + @param dhcp_log_func: A callable accepting the dhclient output and error + streams. @return: A list of dicts of representing the dhcp leases parsed from the dhcp.leases file or empty list. @@ -215,14 +220,20 @@ pid_file = os.path.join(cleandir, 'dhclient.pid') lease_file = os.path.join(cleandir, 'dhcp.leases') + # In some cases files in /var/tmp may not be executable, launching dhclient + # from there will certainly raise 'Permission denied' error. Try launching + # the original dhclient instead. + if not os.access(sandbox_dhclient_cmd, os.X_OK): + sandbox_dhclient_cmd = dhclient_cmd_path + # ISC dhclient needs the interface up to send initial discovery packets. # Generally dhclient relies on dhclient-script PREINIT action to bring the # link up before attempting discovery. Since we are using -sf /bin/true, # we need to do that "link up" ourselves first. - util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) + subp.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] - util.subp(cmd, capture=True) + out, err = subp.subp(cmd, capture=True) # Wait for pid file and lease file to appear, and for the process # named by the pid file to daemonize (have pid 1 as its parent). If we @@ -239,6 +250,7 @@ return [] ppid = 'unknown' + daemonized = False for _ in range(0, 1000): pid_content = util.load_file(pid_file).strip() try: @@ -250,13 +262,17 @@ if ppid == 1: LOG.debug('killing dhclient with pid=%s', pid) os.kill(pid, signal.SIGKILL) - return parse_dhcp_lease_file(lease_file) + daemonized = True + break time.sleep(0.01) - LOG.error( - 'dhclient(pid=%s, parentpid=%s) failed to daemonize after %s seconds', - pid_content, ppid, 0.01 * 1000 - ) + if not daemonized: + LOG.error( + 'dhclient(pid=%s, parentpid=%s) failed to daemonize after %s ' + 'seconds', pid_content, ppid, 0.01 * 1000 + ) + if dhcp_log_func is not None: + dhcp_log_func(out, err) return parse_dhcp_lease_file(lease_file) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/eni.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/eni.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/eni.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/eni.py 2021-05-11 16:34:29.000000000 +0000 @@ -11,6 +11,7 @@ from .network_state import subnet_is_ipv6 from cloudinit import log as logging +from cloudinit import subp from cloudinit import util @@ -386,6 +387,8 @@ if k == 'network': if ':' in route[k]: route_line += ' -A inet6' + elif route.get('prefix') == 32: + route_line += ' -host' else: route_line += ' -net' if 'prefix' in route: @@ -400,6 +403,10 @@ sections = [] subnets = iface.get('subnets', {}) accept_ra = iface.pop('accept-ra', None) + ethernet_wol = iface.pop('wakeonlan', None) + if ethernet_wol: + # Specify WOL setting 'g' for using "Magic Packet" + iface['ethernet-wol'] = 'g' if subnets: for index, subnet in enumerate(subnets): ipv4_subnet_mtu = None @@ -482,10 +489,8 @@ if searchdomains: lo['subnets'][0]["dns_search"] = (" ".join(searchdomains)) - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' + # Apply a sort order to ensure that we write out the physical + # interfaces first; this is critical for bonding order = { 'loopback': 0, 'physical': 1, @@ -511,13 +516,13 @@ return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" def render_network_state(self, network_state, templates=None, target=None): - fpeni = util.target_path(target, self.eni_path) + fpeni = subp.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: - netrules = util.target_path(target, self.netrules_path) + netrules = subp.target_path(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) @@ -544,9 +549,9 @@ expected = ['ifquery', 'ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False - eni = util.target_path(target, 'etc/network/interfaces') + eni = subp.target_path(target, 'etc/network/interfaces') if not os.path.isfile(eni): return False diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/freebsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/freebsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/freebsd.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/freebsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -1,175 +1,59 @@ # This file is part of cloud-init. See LICENSE file for license information. -import re - from cloudinit import log as logging -from cloudinit import net +import cloudinit.net.bsd +from cloudinit import subp from cloudinit import util -from cloudinit.distros import rhel_util -from cloudinit.distros.parsers.resolv_conf import ResolvConf - -from . import renderer LOG = logging.getLogger(__name__) -class Renderer(renderer.Renderer): - resolv_conf_fn = 'etc/resolv.conf' - rc_conf_fn = 'etc/rc.conf' +class Renderer(cloudinit.net.bsd.BSDRenderer): def __init__(self, config=None): - if not config: - config = {} - self.dhcp_interfaces = [] - self._postcmds = config.get('postcmds', True) - - def _update_rc_conf(self, settings, target=None): - fn = util.target_path(target, self.rc_conf_fn) - rhel_util.update_sysconfig_file(fn, settings) - - def _write_ifconfig_entries(self, settings, target=None): - ifname_by_mac = net.get_interfaces_by_mac() - for interface in settings.iter_interfaces(): - device_name = interface.get("name") - device_mac = interface.get("mac_address") - if device_name and re.match(r'^lo\d+$', device_name): - continue - if device_mac not in ifname_by_mac: - LOG.info('Cannot find any device with MAC %s', device_mac) - elif device_mac and device_name: - cur_name = ifname_by_mac[device_mac] - if cur_name != device_name: - LOG.info('netif service will rename interface %s to %s', - cur_name, device_name) - self._update_rc_conf( - {'ifconfig_%s_name' % cur_name: device_name}, - target=target) - else: - device_name = ifname_by_mac[device_mac] + self._route_cpt = 0 + super(Renderer, self).__init__() - LOG.info('Configuring interface %s', device_name) - ifconfig = 'DHCP' # default + def rename_interface(self, cur_name, device_name): + self.set_rc_config_value('ifconfig_%s_name' % cur_name, device_name) - for subnet in interface.get("subnets", []): - if ifconfig != 'DHCP': - LOG.info('The FreeBSD provider only set the first subnet.') - break - if subnet.get('type') == 'static': - if not subnet.get('netmask'): - LOG.debug( - 'Skipping IP %s, because there is no netmask', - subnet.get('address')) - continue - LOG.debug('Configuring dev %s with %s / %s', device_name, - subnet.get('address'), subnet.get('netmask')) - # Configure an ipv4 address. - ifconfig = ( - subnet.get('address') + ' netmask ' + - subnet.get('netmask')) - - if ifconfig == 'DHCP': - self.dhcp_interfaces.append(device_name) - self._update_rc_conf( - {'ifconfig_' + device_name: ifconfig}, - target=target) - - def _write_route_entries(self, settings, target=None): - routes = list(settings.iter_routes()) - for interface in settings.iter_interfaces(): - subnets = interface.get("subnets", []) - for subnet in subnets: - if subnet.get('type') != 'static': - continue - gateway = subnet.get('gateway') - if gateway and len(gateway.split('.')) == 4: - routes.append({ - 'network': '0.0.0.0', - 'netmask': '0.0.0.0', - 'gateway': gateway}) - routes += subnet.get('routes', []) - route_cpt = 0 - for route in routes: - network = route.get('network') - if not network: - LOG.debug('Skipping a bad route entry') - continue - netmask = route.get('netmask') - gateway = route.get('gateway') - route_cmd = "-route %s/%s %s" % (network, netmask, gateway) - if network == '0.0.0.0': - self._update_rc_conf( - {'defaultrouter': gateway}, target=target) + def write_config(self): + for device_name, v in self.interface_configurations.items(): + if isinstance(v, dict): + self.set_rc_config_value( + 'ifconfig_' + device_name, + v.get('address') + ' netmask ' + v.get('netmask')) else: - self._update_rc_conf( - {'route_net%d' % route_cpt: route_cmd}, target=target) - route_cpt += 1 - - def _write_resolve_conf(self, settings, target=None): - nameservers = settings.dns_nameservers - searchdomains = settings.dns_searchdomains - for interface in settings.iter_interfaces(): - for subnet in interface.get("subnets", []): - if 'dns_nameservers' in subnet: - nameservers.extend(subnet['dns_nameservers']) - if 'dns_search' in subnet: - searchdomains.extend(subnet['dns_search']) - # Try to read the /etc/resolv.conf or just start from scratch if that - # fails. - try: - resolvconf = ResolvConf(util.load_file(util.target_path( - target, self.resolv_conf_fn))) - resolvconf.parse() - except IOError: - util.logexc(LOG, "Failed to parse %s, use new empty file", - util.target_path(target, self.resolv_conf_fn)) - resolvconf = ResolvConf('') - resolvconf.parse() - - # Add some nameservers - for server in nameservers: - try: - resolvconf.add_nameserver(server) - except ValueError: - util.logexc(LOG, "Failed to add nameserver %s", server) - - # And add any searchdomains. - for domain in searchdomains: - try: - resolvconf.add_search_domain(domain) - except ValueError: - util.logexc(LOG, "Failed to add search domain %s", domain) - util.write_file( - util.target_path(target, self.resolv_conf_fn), - str(resolvconf), 0o644) - - def _write_network(self, settings, target=None): - self._write_ifconfig_entries(settings, target=target) - self._write_route_entries(settings, target=target) - self._write_resolve_conf(settings, target=target) - - self.start_services(run=self._postcmds) - - def render_network_state(self, network_state, templates=None, target=None): - self._write_network(network_state, target=target) + self.set_rc_config_value('ifconfig_' + device_name, 'DHCP') def start_services(self, run=False): if not run: LOG.debug("freebsd generate postcmd disabled") return - util.subp(['service', 'netif', 'restart'], capture=True) + subp.subp(['service', 'netif', 'restart'], capture=True) # On FreeBSD 10, the restart of routing and dhclient is likely to fail # because # - routing: it cannot remove the loopback route, but it will still set # up the default route as expected. # - dhclient: it cannot stop the dhclient started by the netif service. # In both case, the situation is ok, and we can proceed. - util.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) - for dhcp_interface in self.dhcp_interfaces: - util.subp(['service', 'dhclient', 'restart', dhcp_interface], + subp.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) + + for dhcp_interface in self.dhcp_interfaces(): + subp.subp(['service', 'dhclient', 'restart', dhcp_interface], rcs=[0, 1], capture=True) + def set_route(self, network, netmask, gateway): + if network == '0.0.0.0': + self.set_rc_config_value('defaultrouter', gateway) + else: + route_name = 'route_net%d' % self._route_cpt + route_cmd = "-route %s/%s %s" % (network, netmask, gateway) + self.set_rc_config_value(route_name, route_cmd) + self._route_cpt += 1 + def available(target=None): return util.is_FreeBSD() diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/__init__.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/__init__.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/__init__.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/__init__.py 2021-05-11 16:34:29.000000000 +0000 @@ -6,18 +6,33 @@ # This file is part of cloud-init. See LICENSE file for license information. import errno +import functools +import ipaddress import logging import os import re -from functools import partial -from cloudinit.net.network_state import mask_to_net_prefix +from cloudinit import subp from cloudinit import util +from cloudinit.net.network_state import mask_to_net_prefix from cloudinit.url_helper import UrlError, readurl LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" DEFAULT_PRIMARY_INTERFACE = 'eth0' +OVS_INTERNAL_INTERFACE_LOOKUP_CMD = [ + "ovs-vsctl", + "--format", + "csv", + "--no-headings", + "--timeout", + "10", + "--columns", + "name", + "find", + "interface", + "type=internal", +] def natural_sort_key(s, _nsre=re.compile('([0-9]+)')): @@ -97,10 +112,6 @@ return read_sys_net_safe(devname, "operstate", translate=translate) -def is_wireless(devname): - return os.path.exists(sys_dev_path(devname, "wireless")) - - def is_bridge(devname): return os.path.exists(sys_dev_path(devname, "bridge")) @@ -127,6 +138,61 @@ return (os.path.exists(bonding_path) or os.path.exists(bridge_path)) +def master_is_openvswitch(devname): + """Return a bool indicating if devname's master is openvswitch""" + master_path = get_master(devname) + if master_path is None: + return False + ovs_path = sys_dev_path(devname, path="upper_ovs-system") + return os.path.exists(ovs_path) + + +@functools.lru_cache(maxsize=None) +def openvswitch_is_installed() -> bool: + """Return a bool indicating if Open vSwitch is installed in the system.""" + ret = bool(subp.which("ovs-vsctl")) + if not ret: + LOG.debug( + "ovs-vsctl not in PATH; not detecting Open vSwitch interfaces" + ) + return ret + + +@functools.lru_cache(maxsize=None) +def get_ovs_internal_interfaces() -> list: + """Return a list of the names of OVS internal interfaces on the system. + + These will all be strings, and are used to exclude OVS-specific interface + from cloud-init's network configuration handling. + """ + try: + out, _err = subp.subp(OVS_INTERNAL_INTERFACE_LOOKUP_CMD) + except subp.ProcessExecutionError as exc: + if "database connection failed" in exc.stderr: + LOG.info( + "Open vSwitch is not yet up; no interfaces will be detected as" + " OVS-internal" + ) + return [] + raise + else: + return out.splitlines() + + +def is_openvswitch_internal_interface(devname: str) -> bool: + """Returns True if this is an OVS internal interface. + + If OVS is not installed or not yet running, this will return False. + """ + if not openvswitch_is_installed(): + return False + ovs_bridges = get_ovs_internal_interfaces() + if devname in ovs_bridges: + LOG.debug("Detected %s as an OVS interface", devname) + return True + return False + + def is_netfailover(devname, driver=None): """ netfailover driver uses 3 nics, master, primary and standby. this returns True if the device is either the primary or standby @@ -264,28 +330,6 @@ return 'DEVTYPE=vlan' in uevent.splitlines() -def is_connected(devname): - # is_connected isn't really as simple as that. 2 is - # 'physically connected'. 3 is 'not connected'. but a wlan interface will - # always show 3. - iflink = read_sys_net_safe(devname, "iflink") - if iflink == "2": - return True - if not is_wireless(devname): - return False - LOG.debug("'%s' is wireless, basing 'connected' on carrier", devname) - return read_sys_net_safe(devname, "carrier", - translate={'0': False, '1': True}) - - -def is_physical(devname): - return os.path.exists(sys_dev_path(devname, "device")) - - -def is_present(devname): - return os.path.exists(sys_dev_path(devname)) - - def device_driver(devname): """Return the device driver for net device named 'devname'.""" driver = None @@ -334,10 +378,20 @@ """Return the name of the 'fallback' network device.""" if util.is_FreeBSD(): return find_fallback_nic_on_freebsd(blacklist_drivers) + elif util.is_NetBSD() or util.is_OpenBSD(): + return find_fallback_nic_on_netbsd_or_openbsd(blacklist_drivers) else: return find_fallback_nic_on_linux(blacklist_drivers) +def find_fallback_nic_on_netbsd_or_openbsd(blacklist_drivers=None): + values = list(sorted( + get_interfaces_by_mac().values(), + key=natural_sort_key)) + if values: + return values[0] + + def find_fallback_nic_on_freebsd(blacklist_drivers=None): """Return the name of the 'fallback' network device on FreeBSD. @@ -347,7 +401,7 @@ we'll use the first interface from ``ifconfig -l -u ether`` """ - stdout, _stderr = util.subp(['ifconfig', '-l', '-u', 'ether']) + stdout, _stderr = subp.subp(['ifconfig', '-l', '-u', 'ether']) values = stdout.split() if values: return values[0] @@ -508,43 +562,6 @@ raise RuntimeError('Unknown network config version: %s' % version) -def wait_for_physdevs(netcfg, strict=True): - physdevs = extract_physdevs(netcfg) - - # set of expected iface names and mac addrs - expected_ifaces = dict([(iface[0], iface[1]) for iface in physdevs]) - expected_macs = set(expected_ifaces.keys()) - - # set of current macs - present_macs = get_interfaces_by_mac().keys() - - # compare the set of expected mac address values to - # the current macs present; we only check MAC as cloud-init - # has not yet renamed interfaces and the netcfg may include - # such renames. - for _ in range(0, 5): - if expected_macs.issubset(present_macs): - LOG.debug('net: all expected physical devices present') - return - - missing = expected_macs.difference(present_macs) - LOG.debug('net: waiting for expected net devices: %s', missing) - for mac in missing: - # trigger a settle, unless this interface exists - syspath = sys_dev_path(expected_ifaces[mac]) - settle = partial(util.udevadm_settle, exists=syspath) - msg = 'Waiting for udev events to settle or %s exists' % syspath - util.log_time(LOG.debug, msg, func=settle) - - # update present_macs after settles - present_macs = get_interfaces_by_mac().keys() - - msg = 'Not all expected physical devices present: %s' % missing - LOG.warning(msg) - if strict: - raise RuntimeError(msg) - - def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): """read the network config and rename devices accordingly. if strict_present is false, then do not raise exception if no devices @@ -558,7 +575,9 @@ try: _rename_interfaces(extract_physdevs(netcfg)) except RuntimeError as e: - raise RuntimeError('Failed to apply network config names: %s' % e) + raise RuntimeError( + 'Failed to apply network config names: %s' % e + ) from e def interface_has_own_mac(ifname, strict=False): @@ -609,9 +628,9 @@ if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") - ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', + ipv6, _err = subp.subp(['ip', '-6', 'addr', 'show', 'permanent', 'scope', 'global'], capture=True) - ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) + ipv4, _err = subp.subp(['ip', '-4', 'addr', 'show'], capture=True) nics_with_addresses = set() for bytes_out in (ipv6, ipv4): @@ -647,13 +666,13 @@ for data in cur_info.values()) def rename(cur, new): - util.subp(["ip", "link", "set", cur, "name", new], capture=True) + subp.subp(["ip", "link", "set", cur, "name", new], capture=True) def down(name): - util.subp(["ip", "link", "set", name, "down"], capture=True) + subp.subp(["ip", "link", "set", name, "down"], capture=True) def up(name): - util.subp(["ip", "link", "set", name, "up"], capture=True) + subp.subp(["ip", "link", "set", name, "up"], capture=True) ops = [] errors = [] @@ -796,26 +815,34 @@ return mac -def get_interfaces_by_mac(): +def get_interfaces_by_mac(blacklist_drivers=None) -> dict: if util.is_FreeBSD(): - return get_interfaces_by_mac_on_freebsd() + return get_interfaces_by_mac_on_freebsd( + blacklist_drivers=blacklist_drivers) + elif util.is_NetBSD(): + return get_interfaces_by_mac_on_netbsd( + blacklist_drivers=blacklist_drivers) + elif util.is_OpenBSD(): + return get_interfaces_by_mac_on_openbsd( + blacklist_drivers=blacklist_drivers) else: - return get_interfaces_by_mac_on_linux() + return get_interfaces_by_mac_on_linux( + blacklist_drivers=blacklist_drivers) -def get_interfaces_by_mac_on_freebsd(): - (out, _) = util.subp(['ifconfig', '-a', 'ether']) +def get_interfaces_by_mac_on_freebsd(blacklist_drivers=None) -> dict(): + (out, _) = subp.subp(['ifconfig', '-a', 'ether']) # flatten each interface block in a single line def flatten(out): curr_block = '' - for l in out.split('\n'): - if l.startswith('\t'): - curr_block += l + for line in out.split('\n'): + if line.startswith('\t'): + curr_block += line else: if curr_block: yield curr_block - curr_block = l + curr_block = line yield curr_block # looks for interface and mac in a list of flatten block @@ -830,12 +857,44 @@ return results -def get_interfaces_by_mac_on_linux(): +def get_interfaces_by_mac_on_netbsd(blacklist_drivers=None) -> dict(): + ret = {} + re_field_match = ( + r"(?P\w+).*address:\s" + r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*" + ) + (out, _) = subp.subp(['ifconfig', '-a']) + if_lines = re.sub(r'\n\s+', ' ', out).splitlines() + for line in if_lines: + m = re.match(re_field_match, line) + if m: + fields = m.groupdict() + ret[fields['mac']] = fields['ifname'] + return ret + + +def get_interfaces_by_mac_on_openbsd(blacklist_drivers=None) -> dict(): + ret = {} + re_field_match = ( + r"(?P\w+).*lladdr\s" + r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*") + (out, _) = subp.subp(['ifconfig', '-a']) + if_lines = re.sub(r'\n\s+', ' ', out).splitlines() + for line in if_lines: + m = re.match(re_field_match, line) + if m: + fields = m.groupdict() + ret[fields['mac']] = fields['ifname'] + return ret + + +def get_interfaces_by_mac_on_linux(blacklist_drivers=None) -> dict: """Build a dictionary of tuples {mac: name}. Bridges and any devices that have a 'stolen' mac are excluded.""" ret = {} - for name, mac, _driver, _devid in get_interfaces(): + for name, mac, _driver, _devid in get_interfaces( + blacklist_drivers=blacklist_drivers): if mac in ret: raise RuntimeError( "duplicate mac found! both '%s' and '%s' have mac '%s'" % @@ -853,11 +912,13 @@ return ret -def get_interfaces(): +def get_interfaces(blacklist_drivers=None) -> list: """Return list of interface tuples (name, mac, driver, device_id) Bridges and any devices that have a 'stolen' mac are excluded.""" ret = [] + if blacklist_drivers is None: + blacklist_drivers = [] devs = get_devicelist() # 16 somewhat arbitrarily chosen. Normally a mac is 6 '00:' tokens. zero_mac = ':'.join(('00',) * 16) @@ -870,8 +931,10 @@ continue if is_bond(name): continue - if get_master(name) is not None and not master_is_bridge_or_bond(name): - continue + if get_master(name) is not None: + if (not master_is_bridge_or_bond(name) and + not master_is_openvswitch(name)): + continue if is_netfailover(name): continue mac = get_interface_mac(name) @@ -881,7 +944,13 @@ # skip nics that have no mac (00:00....) if name != 'lo' and mac == zero_mac[:len(mac)]: continue - ret.append((name, mac, device_driver(name), device_devid(name))) + if is_openvswitch_internal_interface(name): + continue + # skip nics that have drivers blacklisted + driver = device_driver(name) + if driver in blacklist_drivers: + continue + ret.append((name, mac, driver, device_devid(name))) return ret @@ -917,6 +986,38 @@ return True +def is_ip_address(s: str) -> bool: + """Returns a bool indicating if ``s`` is an IP address. + + :param s: + The string to test. + + :return: + A bool indicating if the string contains an IP address or not. + """ + try: + ipaddress.ip_address(s) + except ValueError: + return False + return True + + +def is_ipv4_address(s: str) -> bool: + """Returns a bool indicating if ``s`` is an IPv4 address. + + :param s: + The string to test. + + :return: + A bool indicating if the string contains an IPv4 address or not. + """ + try: + ipaddress.IPv4Address(s) + except ValueError: + return False + return True + + class EphemeralIPv4Network(object): """Context manager which sets up temporary static network configuration. @@ -950,7 +1051,8 @@ self.prefix = mask_to_net_prefix(prefix_or_mask) except ValueError as e: raise ValueError( - 'Cannot setup network: {0}'.format(e)) + 'Cannot setup network: {0}'.format(e) + ) from e self.connectivity_url = connectivity_url self.interface = interface @@ -990,11 +1092,11 @@ def __exit__(self, excp_type, excp_value, excp_traceback): """Teardown anything we set up.""" for cmd in self.cleanup_cmds: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _delete_address(self, address, prefix): """Perform the ip command to remove the specified address.""" - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'del', '%s/%s' % (address, prefix), 'dev', self.interface], capture=True) @@ -1006,11 +1108,11 @@ 'Attempting setup of ephemeral network on %s with %s brd %s', self.interface, cidr, self.broadcast) try: - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'add', cidr, 'broadcast', self.broadcast, 'dev', self.interface], capture=True, update_env={'LANG': 'C'}) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if "File exists" not in e.stderr: raise LOG.debug( @@ -1018,7 +1120,7 @@ self.interface, self.ip) else: # Address creation success, bring up device and queue cleanup - util.subp( + subp.subp( ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface, 'up'], capture=True) self.cleanup_cmds.append( @@ -1033,9 +1135,9 @@ # ("0.0.0.0/0", "130.56.240.1")] for net_address, gateway in self.static_routes: via_arg = [] - if gateway != "0.0.0.0/0": + if gateway != "0.0.0.0": via_arg = ['via', gateway] - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', net_address] + via_arg + ['dev', self.interface], capture=True) self.cleanup_cmds.insert( @@ -1045,20 +1147,20 @@ def _bringup_router(self): """Perform the ip commands to fully setup the router if needed.""" # Check if a default route exists and exit if it does - out, _ = util.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) + out, _ = subp.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) if 'default' in out: LOG.debug( 'Skip ephemeral route setup. %s already has default route: %s', self.interface, out.strip()) return - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface, 'src', self.ip], capture=True) self.cleanup_cmds.insert( 0, ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface, 'src', self.ip]) - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', 'default', 'via', self.router, 'dev', self.interface], capture=True) self.cleanup_cmds.insert( diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/netbsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/netbsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/netbsd.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/netbsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,44 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import log as logging +from cloudinit import subp +from cloudinit import util +import cloudinit.net.bsd + +LOG = logging.getLogger(__name__) + + +class Renderer(cloudinit.net.bsd.BSDRenderer): + + def __init__(self, config=None): + super(Renderer, self).__init__() + + def write_config(self): + if self.dhcp_interfaces(): + self.set_rc_config_value('dhcpcd', 'YES') + self.set_rc_config_value( + 'dhcpcd_flags', + ' '.join(self.dhcp_interfaces()) + ) + for device_name, v in self.interface_configurations.items(): + if isinstance(v, dict): + self.set_rc_config_value( + 'ifconfig_' + device_name, + v.get('address') + ' netmask ' + v.get('netmask')) + + def start_services(self, run=False): + if not run: + LOG.debug("netbsd generate postcmd disabled") + return + + subp.subp(['service', 'network', 'restart'], capture=True) + if self.dhcp_interfaces(): + subp.subp(['service', 'dhcpcd', 'restart'], capture=True) + + def set_route(self, network, netmask, gateway): + if network == '0.0.0.0': + self.set_rc_config_value('defaultroute', gateway) + + +def available(target=None): + return util.is_NetBSD() diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/netplan.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/netplan.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/netplan.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/netplan.py 2021-05-11 16:34:29.000000000 +0000 @@ -8,6 +8,7 @@ from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit import safeyaml from cloudinit.net import SYS_CLASS_NET, get_devicelist @@ -164,14 +165,14 @@ def _clean_default(target=None): # clean out any known default files and derived files in target # LP: #1675576 - tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml") + tpath = subp.target_path(target, "etc/netplan/00-snapd-config.yaml") if not os.path.isfile(tpath): return content = util.load_file(tpath, decode=False) if content != KNOWN_SNAPD_CONFIG: return - derived = [util.target_path(target, f) for f in ( + derived = [subp.target_path(target, f) for f in ( 'run/systemd/network/10-netplan-all-en.network', 'run/systemd/network/10-netplan-all-eth.network', 'run/systemd/generator/netplan.stamp')] @@ -203,10 +204,10 @@ def features(self): if self._features is None: try: - info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True) + info_blob, _err = subp.subp(self.NETPLAN_INFO, capture=True) info = util.load_yaml(info_blob) self._features = info['netplan.io']['features'] - except util.ProcessExecutionError: + except subp.ProcessExecutionError: # if the info subcommand is not present then we don't have any # new features pass @@ -218,7 +219,7 @@ # check network state for version # if v2, then extract network_state.config # else render_v2_from_state - fpnplan = os.path.join(util.target_path(target), self.netplan_path) + fpnplan = os.path.join(subp.target_path(target), self.netplan_path) util.ensure_dir(os.path.dirname(fpnplan)) header = self.netplan_header if self.netplan_header else "" @@ -239,7 +240,7 @@ if not run: LOG.debug("netplan generate postcmd disabled") return - util.subp(self.NETPLAN_GENERATE, capture=True) + subp.subp(self.NETPLAN_GENERATE, capture=True) def _net_setup_link(self, run=False): """To ensure device link properties are applied, we poke @@ -253,7 +254,7 @@ for cmd in [setup_lnk + [SYS_CLASS_NET + iface] for iface in get_devicelist() if os.path.islink(SYS_CLASS_NET + iface)]: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _render_content(self, network_state): @@ -406,7 +407,7 @@ expected = ['netplan'] search = ['/usr/sbin', '/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False return True diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/network_state.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/network_state.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/network_state.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/network_state.py 2021-05-11 16:34:29.000000000 +0000 @@ -215,7 +215,7 @@ return ( route.get('prefix') == 0 and route.get('network') in default_nets - ) + ) class NetworkStateInterpreter(metaclass=CommandHandlerMeta): @@ -297,9 +297,10 @@ command_type = command['type'] try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) except InvalidCommand: @@ -312,13 +313,14 @@ def parse_config_v2(self, skip_broken=True): for command_type, command in self._config.items(): - if command_type == 'version': + if command_type in ['version', 'renderer']: continue try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) self._v2_common(command) @@ -367,6 +369,9 @@ accept_ra = command.get('accept-ra', None) if accept_ra is not None: accept_ra = util.is_true(accept_ra) + wakeonlan = command.get('wakeonlan', None) + if wakeonlan is not None: + wakeonlan = util.is_true(wakeonlan) iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -377,7 +382,8 @@ 'address': None, 'gateway': None, 'subnets': subnets, - 'accept-ra': accept_ra + 'accept-ra': accept_ra, + 'wakeonlan': wakeonlan, }) self._network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @@ -696,7 +702,7 @@ def handle_wifis(self, command): LOG.warning('Wifi configuration is only available to distros with' - 'netplan rendering support.') + ' netplan rendering support.') def _v2_common(self, cfg): LOG.debug('v2_common: handling config:\n%s', cfg) @@ -722,10 +728,10 @@ item_params = dict((key, value) for (key, value) in item_cfg.items() if key not in NETWORK_V2_KEY_FILTER) - # we accept the fixed spelling, but write the old for compatability + # we accept the fixed spelling, but write the old for compatibility # Xenial does not have an updated netplan which supports the # correct spelling. LP: #1756701 - params = item_params['parameters'] + params = item_params.get('parameters', {}) grat_value = params.pop('gratuitous-arp', None) if grat_value: params['gratuitious-arp'] = grat_value @@ -734,8 +740,7 @@ 'type': cmd_type, 'name': item_name, cmd_type + '_interfaces': item_cfg.get('interfaces'), - 'params': dict((v2key_to_v1[k], v) for k, v in - item_params.get('parameters', {}).items()) + 'params': dict((v2key_to_v1[k], v) for k, v in params.items()) } if 'mtu' in item_cfg: v1_cmd['mtu'] = item_cfg['mtu'] @@ -819,7 +824,8 @@ if subnet.get('type') in ('static', 'static6'): normal_subnet.update( - _normalize_net_keys(normal_subnet, address_keys=('address',))) + _normalize_net_keys(normal_subnet, address_keys=( + 'address', 'ip_address',))) normal_subnet['routes'] = [_normalize_route(r) for r in subnet.get('routes', [])] @@ -915,9 +921,10 @@ if metric: try: normal_route['metric'] = int(metric) - except ValueError: + except ValueError as e: raise TypeError( - 'Route config metric {} is not an integer'.format(metric)) + 'Route config metric {} is not an integer'.format(metric) + ) from e return normal_route diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/openbsd.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/openbsd.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/openbsd.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/openbsd.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,46 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import log as logging +from cloudinit import subp +from cloudinit import util +import cloudinit.net.bsd + +LOG = logging.getLogger(__name__) + + +class Renderer(cloudinit.net.bsd.BSDRenderer): + + def write_config(self): + for device_name, v in self.interface_configurations.items(): + if_file = 'etc/hostname.{}'.format(device_name) + fn = subp.target_path(self.target, if_file) + if device_name in self.dhcp_interfaces(): + content = 'dhcp\n' + elif isinstance(v, dict): + try: + content = "inet {address} {netmask}\n".format( + address=v['address'], + netmask=v['netmask'] + ) + except KeyError: + LOG.error( + "Invalid static configuration for %s", + device_name) + util.write_file(fn, content) + + def start_services(self, run=False): + if not self._postcmds: + LOG.debug("openbsd generate postcmd disabled") + return + subp.subp(['sh', '/etc/netstart'], capture=True) + + def set_route(self, network, netmask, gateway): + if network == '0.0.0.0': + if_file = 'etc/mygate' + fn = subp.target_path(self.target, if_file) + content = gateway + '\n' + util.write_file(fn, content) + + +def available(target=None): + return util.is_OpenBSD() diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/renderers.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/renderers.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/renderers.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/renderers.py 2021-05-11 16:34:29.000000000 +0000 @@ -2,18 +2,23 @@ from . import eni from . import freebsd +from . import netbsd from . import netplan from . import RendererNotFoundError +from . import openbsd from . import sysconfig NAME_TO_RENDERER = { "eni": eni, "freebsd": freebsd, + "netbsd": netbsd, "netplan": netplan, + "openbsd": openbsd, "sysconfig": sysconfig, } -DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan", "freebsd"] +DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan", "freebsd", + "netbsd", "openbsd"] def search(priority=None, target=None, first=False): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/sysconfig.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/sysconfig.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/sysconfig.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/sysconfig.py 2021-05-11 16:34:29.000000000 +0000 @@ -9,6 +9,7 @@ from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros.parsers import networkmanager_conf from cloudinit.distros.parsers import resolv_conf @@ -18,7 +19,7 @@ LOG = logging.getLogger(__name__) NM_CFG_FILE = "/etc/NetworkManager/NetworkManager.conf" -KNOWN_DISTROS = ['centos', 'fedora', 'rhel', 'suse'] +KNOWN_DISTROS = ['almalinux', 'centos', 'fedora', 'rhel', 'suse'] def _make_header(sep='#'): @@ -98,6 +99,10 @@ def __len__(self): return len(self._conf) + def skip_key_value(self, key, val): + """Skip the pair key, value if it matches a certain rule.""" + return False + def to_string(self): buf = io.StringIO() buf.write(_make_header()) @@ -105,6 +110,8 @@ buf.write("\n") for key in sorted(self._conf.keys()): value = self._conf[key] + if self.skip_key_value(key, value): + continue if isinstance(value, bool): value = self._bool_map[value] if not isinstance(value, str): @@ -213,6 +220,7 @@ 'bond': 'Bond', 'bridge': 'Bridge', 'infiniband': 'InfiniBand', + 'vlan': 'Vlan', } def __init__(self, iface_name, base_sysconf_dir, templates, @@ -266,6 +274,11 @@ c.routes = self.routes.copy() return c + def skip_key_value(self, key, val): + if key == 'TYPE' and val == 'Vlan': + return True + return False + class Renderer(renderer.Renderer): """Renders network information in a /etc/sysconfig format.""" @@ -300,7 +313,8 @@ } # If these keys exist, then their values will be used to form - # a BONDING_OPTS grouping; otherwise no grouping will be set. + # a BONDING_OPTS / BONDING_MODULE_OPTS grouping; otherwise no + # grouping will be set. bond_tpl_opts = tuple([ ('bond_mode', "mode=%s"), ('bond_xmit_hash_policy', "xmit_hash_policy=%s"), @@ -354,6 +368,11 @@ if new_key: iface_cfg[new_key] = old_value + # only set WakeOnLan for physical interfaces + if ('wakeonlan' in iface and iface['wakeonlan'] and + iface['type'] == 'physical'): + iface_cfg['ETHTOOL_OPTS'] = 'wol g' + @classmethod def _render_subnets(cls, iface_cfg, subnets, has_default_route, flavor): # setting base values @@ -378,6 +397,13 @@ # Only IPv6 is DHCP, IPv4 may be static iface_cfg['BOOTPROTO'] = 'dhcp6' iface_cfg['DHCLIENT6_MODE'] = 'managed' + # only if rhel AND dhcpv6 stateful + elif (flavor == 'rhel' and + subnet_type == 'ipv6_dhcpv6-stateful'): + iface_cfg['BOOTPROTO'] = 'dhcp' + iface_cfg['DHCPV6C'] = True + iface_cfg['IPV6INIT'] = True + iface_cfg['IPV6_AUTOCONF'] = False else: iface_cfg['IPV6INIT'] = True # Configure network settings using DHCPv6 @@ -450,6 +476,10 @@ iface_cfg[mtu_key] = subnet['mtu'] else: iface_cfg[mtu_key] = subnet['mtu'] + + if subnet_is_ipv6(subnet) and flavor == 'rhel': + iface_cfg['IPV6_FORCE_ACCEPT_RA'] = False + iface_cfg['IPV6_AUTOCONF'] = False elif subnet_type == 'manual': if flavor == 'suse': LOG.debug('Unknown subnet type setting "%s"', subnet_type) @@ -504,7 +534,7 @@ iface_cfg['IPADDR6_%d' % ipv6_index] = ipv6_cidr else: iface_cfg['IPV6ADDR_SECONDARIES'] += \ - " " + ipv6_cidr + " " + ipv6_cidr else: ipv4_index = ipv4_index + 1 suff = "" if ipv4_index == 0 else str(ipv4_index) @@ -593,7 +623,7 @@ route_cfg[new_key] = route[old_key] @classmethod - def _render_bonding_opts(cls, iface_cfg, iface): + def _render_bonding_opts(cls, iface_cfg, iface, flavor): bond_opts = [] for (bond_key, value_tpl) in cls.bond_tpl_opts: # Seems like either dash or underscore is possible? @@ -606,7 +636,18 @@ bond_opts.append(value_tpl % (bond_value)) break if bond_opts: - iface_cfg['BONDING_OPTS'] = " ".join(bond_opts) + if flavor == 'suse': + # suse uses the sysconfig support which requires + # BONDING_MODULE_OPTS see + # https://www.kernel.org/doc/Documentation/networking/bonding.txt + # 3.1 Configuration with Sysconfig Support + iface_cfg['BONDING_MODULE_OPTS'] = " ".join(bond_opts) + else: + # rhel uses initscript support and thus requires BONDING_OPTS + # this is also the old default see + # https://www.kernel.org/doc/Documentation/networking/bonding.txt + # 3.2 Configuration with Initscripts Support + iface_cfg['BONDING_OPTS'] = " ".join(bond_opts) @classmethod def _render_physical_interfaces( @@ -634,7 +675,7 @@ for iface in network_state.iter_interfaces(bond_filter): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] - cls._render_bonding_opts(iface_cfg, iface) + cls._render_bonding_opts(iface_cfg, iface, flavor) # Ensure that the master interface (and any of its children) # are actually marked as being bond types... @@ -696,7 +737,16 @@ iface_cfg['ETHERDEVICE'] = iface_name[:iface_name.rfind('.')] else: iface_cfg['VLAN'] = True - iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')] + iface_cfg.kind = 'vlan' + + rdev = iface['vlan-raw-device'] + supported = _supported_vlan_names(rdev, iface['vlan_id']) + if iface_name not in supported: + LOG.info( + "Name '%s' for vlan '%s' is not officially supported" + "by RHEL. Supported: %s", + iface_name, rdev, ' '.join(supported)) + iface_cfg['PHYSDEV'] = rdev iface_subnets = iface.get("subnets", []) route_cfg = iface_cfg.routes @@ -858,19 +908,19 @@ if not templates: templates = self.templates file_mode = 0o644 - base_sysconf_dir = util.target_path(target, self.sysconf_dir) + base_sysconf_dir = subp.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state, self.flavor, templates=templates).items(): util.write_file(path, data, file_mode) if self.dns_path: - dns_path = util.target_path(target, self.dns_path) + dns_path = subp.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) if resolv_content: util.write_file(dns_path, resolv_content, file_mode) if self.networkmanager_conf_path: - nm_conf_path = util.target_path(target, + nm_conf_path = subp.target_path(target, self.networkmanager_conf_path) nm_conf_content = self._render_networkmanager_conf(network_state, templates) @@ -878,12 +928,12 @@ util.write_file(nm_conf_path, nm_conf_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - netrules_path = util.target_path(target, self.netrules_path) + netrules_path = subp.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content, file_mode) if available_nm(target=target): - enable_ifcfg_rh(util.target_path(target, path=NM_CFG_FILE)) + enable_ifcfg_rh(subp.target_path(target, path=NM_CFG_FILE)) - sysconfig_path = util.target_path(target, templates.get('control')) + sysconfig_path = subp.target_path(target, templates.get('control')) # Distros configuring /etc/sysconfig/network as a file e.g. Centos if sysconfig_path.endswith('network'): util.ensure_dir(os.path.dirname(sysconfig_path)) @@ -895,6 +945,15 @@ "\n".join(netcfg) + "\n", file_mode) +def _supported_vlan_names(rdev, vid): + """Return list of supported names for vlan devices per RHEL doc + 11.5. Naming Scheme for VLAN Interfaces.""" + return [ + v.format(rdev=rdev, vid=int(vid)) + for v in ("{rdev}{vid:04}", "{rdev}{vid}", + "{rdev}.{vid:04}", "{rdev}.{vid}")] + + def available(target=None): sysconfig = available_sysconfig(target=target) nm = available_nm(target=target) @@ -906,20 +965,20 @@ expected = ['ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False expected_paths = [ 'etc/sysconfig/network-scripts/network-functions', 'etc/sysconfig/config'] for p in expected_paths: - if os.path.isfile(util.target_path(target, p)): + if os.path.isfile(subp.target_path(target, p)): return True return False def available_nm(target=None): - if not os.path.isfile(util.target_path(target, path=NM_CFG_FILE)): + if not os.path.isfile(subp.target_path(target, path=NM_CFG_FILE)): return False return True diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/tests/test_dhcp.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/tests/test_dhcp.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/tests/test_dhcp.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/tests/test_dhcp.py 2021-05-11 16:34:29.000000000 +0000 @@ -62,7 +62,7 @@ {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) class TestDHCPRFC3442(CiTestCase): @@ -88,7 +88,7 @@ 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) def test_parse_lease_finds_classless_static_routes(self): """ @@ -114,7 +114,7 @@ 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @@ -194,6 +194,11 @@ self.assertEqual([('0.0.0.0/0', '130.56.240.1')], parse_static_routes(rfc3442)) + def test_unspecified_gateway(self): + rfc3442 = "32,169,254,169,254,0,0,0,0" + self.assertEqual([('169.254.169.254/32', '0.0.0.0')], + parse_static_routes(rfc3442)) + def test_parse_static_routes_class_c_b_a(self): class_c = "24,192,168,74,192,168,0,4" class_b = "16,172,16,172,16,0,4" @@ -211,7 +216,7 @@ "class_b": "16,172,16,10", "class_a": "8,10,10", "gateway": "0,0", - "netlen": "33,0", + "netlen": "33,0", } for rfc3442 in bad_rfc3442.values(): self.assertEqual([], parse_static_routes(rfc3442)) @@ -266,7 +271,7 @@ 'Skip dhcp_discovery: nic idontexist not found in get_devicelist.', self.logs.getvalue()) - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_absent_dhclient_command(self, m_fallback, m_which): """When dhclient doesn't exist in the OS, log the issue and no-op.""" @@ -279,7 +284,7 @@ @mock.patch('cloudinit.temp_utils.os.getuid') @mock.patch('cloudinit.net.dhcp.dhcp_discovery') - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid): """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery.""" @@ -302,13 +307,14 @@ @mock.patch('time.sleep', mock.MagicMock()) @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_warns_invalid_pid(self, m_subp, m_kill): """dhcp_discovery logs a warning when pidfile contains invalid content. Lease processing still occurs and no proc kill is attempted. """ + m_subp.return_value = ('', '') tmpdir = self.tmp_dir() dhclient_script = os.path.join(tmpdir, 'dhclient.orig') script_content = '#!/bin/bash\necho fake-dhclient' @@ -324,7 +330,7 @@ """) write_file(self.tmp_path('dhcp.leases', tmpdir), lease_content) - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) @@ -337,13 +343,14 @@ @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') @mock.patch('cloudinit.net.dhcp.util.wait_for_files') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_waits_on_lease_and_pid(self, m_subp, m_wait, m_kill, m_getppid): """dhcp_discovery waits for the presence of pidfile and dhcp.leases.""" + m_subp.return_value = ('', '') tmpdir = self.tmp_dir() dhclient_script = os.path.join(tmpdir, 'dhclient.orig') script_content = '#!/bin/bash\necho fake-dhclient' @@ -364,12 +371,13 @@ @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill, m_getppid): """dhcp_discovery brings up the interface and runs dhclient. It also returns the parsed dhcp.leases file generated in the sandbox. """ + m_subp.return_value = ('', '') tmpdir = self.tmp_dir() dhclient_script = os.path.join(tmpdir, 'dhclient.orig') script_content = '#!/bin/bash\necho fake-dhclient' @@ -389,7 +397,7 @@ write_file(pid_file, "%d\n" % my_pid) m_getppid.return_value = 1 # Indicate that dhclient has daemonized - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) @@ -406,6 +414,87 @@ 'eth9', '-sf', '/bin/true'], capture=True)]) m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)]) + @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') + @mock.patch('cloudinit.net.dhcp.os.kill') + @mock.patch('cloudinit.net.dhcp.subp.subp') + def test_dhcp_discovery_outside_sandbox(self, m_subp, m_kill, m_getppid): + """dhcp_discovery brings up the interface and runs dhclient. + + It also returns the parsed dhcp.leases file generated in the sandbox. + """ + m_subp.return_value = ('', '') + tmpdir = self.tmp_dir() + dhclient_script = os.path.join(tmpdir, 'dhclient.orig') + script_content = '#!/bin/bash\necho fake-dhclient' + write_file(dhclient_script, script_content, mode=0o755) + lease_content = dedent(""" + lease { + interface "eth9"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + } + """) + lease_file = os.path.join(tmpdir, 'dhcp.leases') + write_file(lease_file, lease_content) + pid_file = os.path.join(tmpdir, 'dhclient.pid') + my_pid = 1 + write_file(pid_file, "%d\n" % my_pid) + m_getppid.return_value = 1 # Indicate that dhclient has daemonized + + with mock.patch('os.access', return_value=False): + self.assertCountEqual( + [{'interface': 'eth9', 'fixed-address': '192.168.2.74', + 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], + dhcp_discovery(dhclient_script, 'eth9', tmpdir)) + # dhclient script got copied + with open(os.path.join(tmpdir, 'dhclient.orig')) as stream: + self.assertEqual(script_content, stream.read()) + # Interface was brought up before dhclient called from sandbox + m_subp.assert_has_calls([ + mock.call( + ['ip', 'link', 'set', 'dev', 'eth9', 'up'], capture=True), + mock.call( + [os.path.join(tmpdir, 'dhclient.orig'), '-1', '-v', '-lf', + lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'), + 'eth9', '-sf', '/bin/true'], capture=True)]) + m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)]) + + @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') + @mock.patch('cloudinit.net.dhcp.os.kill') + @mock.patch('cloudinit.net.dhcp.subp.subp') + def test_dhcp_output_error_stream(self, m_subp, m_kill, m_getppid): + """"dhcp_log_func is called with the output and error streams of + dhclinet when the callable is passed.""" + dhclient_err = 'FAKE DHCLIENT ERROR' + dhclient_out = 'FAKE DHCLIENT OUT' + m_subp.return_value = (dhclient_out, dhclient_err) + tmpdir = self.tmp_dir() + dhclient_script = os.path.join(tmpdir, 'dhclient.orig') + script_content = '#!/bin/bash\necho fake-dhclient' + write_file(dhclient_script, script_content, mode=0o755) + lease_content = dedent(""" + lease { + interface "eth9"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + } + """) + lease_file = os.path.join(tmpdir, 'dhcp.leases') + write_file(lease_file, lease_content) + pid_file = os.path.join(tmpdir, 'dhclient.pid') + my_pid = 1 + write_file(pid_file, "%d\n" % my_pid) + m_getppid.return_value = 1 # Indicate that dhclient has daemonized + + def dhcp_log_func(out, err): + self.assertEqual(out, dhclient_out) + self.assertEqual(err, dhclient_err) + + dhcp_discovery( + dhclient_script, 'eth9', tmpdir, dhcp_log_func=dhcp_log_func) + class TestSystemdParseLeases(CiTestCase): @@ -529,7 +618,7 @@ # Ensure that no teardown happens: m_dhcp.assert_not_called() - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_ephemeral_dhcp_setup_network_if_url_connectivity( self, m_dhcp, m_subp): diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/tests/test_init.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/tests/test_init.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/tests/test_init.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/tests/test_init.py 2021-05-11 16:34:29.000000000 +0000 @@ -2,16 +2,20 @@ import copy import errno -import httpretty +import ipaddress import os -import requests import textwrap from unittest import mock +import httpretty +import pytest +import requests + import cloudinit.net as net -from cloudinit.util import ensure_file, write_file, ProcessExecutionError -from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase from cloudinit import safeyaml as yaml +from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase +from cloudinit.subp import ProcessExecutionError +from cloudinit.util import ensure_file, write_file class TestSysDevPath(CiTestCase): @@ -139,12 +143,6 @@ write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state) self.assertFalse(net.is_up('eth0')) - def test_is_wireless(self): - """is_wireless is True when /sys/net/devname/wireless exists.""" - self.assertFalse(net.is_wireless('eth0')) - ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless')) - self.assertTrue(net.is_wireless('eth0')) - def test_is_bridge(self): """is_bridge is True when /sys/net/devname/bridge exists.""" self.assertFalse(net.is_bridge('eth0')) @@ -192,6 +190,28 @@ self.assertTrue(net.master_is_bridge_or_bond('eth1')) self.assertTrue(net.master_is_bridge_or_bond('eth2')) + def test_master_is_openvswitch(self): + ovs_mac = 'bb:cc:aa:bb:cc:aa' + + # No master => False + write_file(os.path.join(self.sysdir, 'eth1', 'address'), ovs_mac) + + self.assertFalse(net.master_is_bridge_or_bond('eth1')) + + # masters without ovs-system => False + write_file(os.path.join(self.sysdir, 'ovs-system', 'address'), ovs_mac) + + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth1', + 'master')) + + self.assertFalse(net.master_is_openvswitch('eth1')) + + # masters with ovs-system => True + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth1', + 'upper_ovs-system')) + + self.assertTrue(net.master_is_openvswitch('eth1')) + def test_is_vlan(self): """is_vlan is True when /sys/net/devname/uevent has DEVTYPE=vlan.""" ensure_file(os.path.join(self.sysdir, 'eth0', 'uevent')) @@ -200,32 +220,6 @@ write_file(os.path.join(self.sysdir, 'eth0', 'uevent'), content) self.assertTrue(net.is_vlan('eth0')) - def test_is_connected_when_physically_connected(self): - """is_connected is True when /sys/net/devname/iflink reports 2.""" - self.assertFalse(net.is_connected('eth0')) - write_file(os.path.join(self.sysdir, 'eth0', 'iflink'), "2") - self.assertTrue(net.is_connected('eth0')) - - def test_is_connected_when_wireless_and_carrier_active(self): - """is_connected is True if wireless /sys/net/devname/carrier is 1.""" - self.assertFalse(net.is_connected('eth0')) - ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless')) - self.assertFalse(net.is_connected('eth0')) - write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), "1") - self.assertTrue(net.is_connected('eth0')) - - def test_is_physical(self): - """is_physical is True when /sys/net/devname/device exists.""" - self.assertFalse(net.is_physical('eth0')) - ensure_file(os.path.join(self.sysdir, 'eth0', 'device')) - self.assertTrue(net.is_physical('eth0')) - - def test_is_present(self): - """is_present is True when /sys/net/devname exists.""" - self.assertFalse(net.is_present('eth0')) - ensure_file(os.path.join(self.sysdir, 'eth0', 'device')) - self.assertTrue(net.is_present('eth0')) - class TestGenerateFallbackConfig(CiTestCase): @@ -341,8 +335,6 @@ class TestNetFindFallBackNic(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFindFallBackNic, self).setUp() sys_mock = mock.patch('cloudinit.net.get_sys_class_path') @@ -396,9 +388,13 @@ """get_devicelist returns a directory listing for SYS_CLASS_NET.""" write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up') write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up') - self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist()) + self.assertCountEqual(['eth0', 'eth1'], net.get_devicelist()) +@mock.patch( + "cloudinit.net.is_openvswitch_internal_interface", + mock.Mock(return_value=False), +) class TestGetInterfaceMAC(CiTestCase): def setUp(self): @@ -495,20 +491,32 @@ ): bridge_mac = 'aa:bb:cc:aa:bb:cc' bond_mac = 'cc:bb:aa:cc:bb:aa' + ovs_mac = 'bb:cc:aa:bb:cc:aa' + write_file(os.path.join(self.sysdir, 'br0', 'address'), bridge_mac) write_file(os.path.join(self.sysdir, 'br0', 'bridge'), '') write_file(os.path.join(self.sysdir, 'bond0', 'address'), bond_mac) write_file(os.path.join(self.sysdir, 'bond0', 'bonding'), '') + write_file(os.path.join(self.sysdir, 'ovs-system', 'address'), + ovs_mac) + write_file(os.path.join(self.sysdir, 'eth1', 'address'), bridge_mac) os.symlink('../br0', os.path.join(self.sysdir, 'eth1', 'master')) write_file(os.path.join(self.sysdir, 'eth2', 'address'), bond_mac) os.symlink('../bond0', os.path.join(self.sysdir, 'eth2', 'master')) + write_file(os.path.join(self.sysdir, 'eth3', 'address'), ovs_mac) + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth3', + 'master')) + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth3', + 'upper_ovs-system')) + interface_names = [interface[0] for interface in net.get_interfaces()] - self.assertEqual(['eth1', 'eth2'], sorted(interface_names)) + self.assertEqual(['eth1', 'eth2', 'eth3', 'ovs-system'], + sorted(interface_names)) class TestInterfaceHasOwnMAC(CiTestCase): @@ -540,7 +548,7 @@ net.interface_has_own_mac('eth1', strict=True) -@mock.patch('cloudinit.net.util.subp') +@mock.patch('cloudinit.net.subp.subp') class TestEphemeralIPV4Network(CiTestCase): with_logs = True @@ -698,19 +706,23 @@ def test_ephemeral_ipv4_network_with_rfc3442_static_routes(self, m_subp): params = { 'interface': 'eth0', 'ip': '192.168.2.2', - 'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255', - 'static_routes': [('169.254.169.254/32', '192.168.2.1'), + 'prefix_or_mask': '255.255.255.255', 'broadcast': '192.168.2.255', + 'static_routes': [('192.168.2.1/32', '0.0.0.0'), + ('169.254.169.254/32', '192.168.2.1'), ('0.0.0.0/0', '192.168.2.1')], 'router': '192.168.2.1'} expected_setup_calls = [ mock.call( - ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24', + ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/32', 'broadcast', '192.168.2.255', 'dev', 'eth0'], capture=True, update_env={'LANG': 'C'}), mock.call( ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'up'], capture=True), mock.call( + ['ip', '-4', 'route', 'add', '192.168.2.1/32', + 'dev', 'eth0'], capture=True), + mock.call( ['ip', '-4', 'route', 'add', '169.254.169.254/32', 'via', '192.168.2.1', 'dev', 'eth0'], capture=True), mock.call( @@ -724,11 +736,14 @@ ['ip', '-4', 'route', 'del', '169.254.169.254/32', 'via', '192.168.2.1', 'dev', 'eth0'], capture=True), mock.call( + ['ip', '-4', 'route', 'del', '192.168.2.1/32', + 'dev', 'eth0'], capture=True), + mock.call( ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'down'], capture=True), mock.call( ['ip', '-family', 'inet', 'addr', 'del', - '192.168.2.2/24', 'dev', 'eth0'], capture=True) + '192.168.2.2/32', 'dev', 'eth0'], capture=True) ] with net.EphemeralIPv4Network(**params): self.assertEqual(expected_setup_calls, m_subp.call_args_list) @@ -993,86 +1008,8 @@ net.extract_physdevs({'version': 3, 'awesome_config': []}) -class TestWaitForPhysdevs(CiTestCase): - - with_logs = True - - def setUp(self): - super(TestWaitForPhysdevs, self).setUp() - self.add_patch('cloudinit.net.get_interfaces_by_mac', - 'm_get_iface_mac') - self.add_patch('cloudinit.util.udevadm_settle', 'm_udev_settle') - - def test_wait_for_physdevs_skips_settle_if_all_present(self): - physdevs = [ - ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'], - ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'], - ] - netcfg = { - 'version': 2, - 'ethernets': {args[1]: _mk_v2_phys(*args) - for args in physdevs}, - } - self.m_get_iface_mac.side_effect = iter([ - {'aa:bb:cc:dd:ee:ff': 'eth0', - '00:11:22:33:44:55': 'ens3'}, - ]) - net.wait_for_physdevs(netcfg) - self.assertEqual(0, self.m_udev_settle.call_count) - - def test_wait_for_physdevs_calls_udev_settle_on_missing(self): - physdevs = [ - ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'], - ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'], - ] - netcfg = { - 'version': 2, - 'ethernets': {args[1]: _mk_v2_phys(*args) - for args in physdevs}, - } - self.m_get_iface_mac.side_effect = iter([ - {'aa:bb:cc:dd:ee:ff': 'eth0'}, # first call ens3 is missing - {'aa:bb:cc:dd:ee:ff': 'eth0', - '00:11:22:33:44:55': 'ens3'}, # second call has both - ]) - net.wait_for_physdevs(netcfg) - self.m_udev_settle.assert_called_with(exists=net.sys_dev_path('ens3')) - - def test_wait_for_physdevs_raise_runtime_error_if_missing_and_strict(self): - physdevs = [ - ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'], - ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'], - ] - netcfg = { - 'version': 2, - 'ethernets': {args[1]: _mk_v2_phys(*args) - for args in physdevs}, - } - self.m_get_iface_mac.return_value = {} - with self.assertRaises(RuntimeError): - net.wait_for_physdevs(netcfg) - - self.assertEqual(5 * len(physdevs), self.m_udev_settle.call_count) - - def test_wait_for_physdevs_no_raise_if_not_strict(self): - physdevs = [ - ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'], - ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'], - ] - netcfg = { - 'version': 2, - 'ethernets': {args[1]: _mk_v2_phys(*args) - for args in physdevs}, - } - self.m_get_iface_mac.return_value = {} - net.wait_for_physdevs(netcfg, strict=False) - self.assertEqual(5 * len(physdevs), self.m_udev_settle.call_count) - - class TestNetFailOver(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFailOver, self).setUp() self.add_patch('cloudinit.net.util', 'm_util') @@ -1297,4 +1234,163 @@ m_standby.return_value = False self.assertFalse(net.is_netfailover(devname, driver)) + +class TestOpenvswitchIsInstalled: + """Test cloudinit.net.openvswitch_is_installed. + + Uses the ``clear_lru_cache`` local autouse fixture to allow us to test + despite the ``lru_cache`` decorator on the unit under test. + """ + + @pytest.fixture(autouse=True) + def clear_lru_cache(self): + net.openvswitch_is_installed.cache_clear() + + @pytest.mark.parametrize( + "expected,which_return", [(True, "/some/path"), (False, None)] + ) + @mock.patch("cloudinit.net.subp.which") + def test_mirrors_which_result(self, m_which, expected, which_return): + m_which.return_value = which_return + assert expected == net.openvswitch_is_installed() + + @mock.patch("cloudinit.net.subp.which") + def test_only_calls_which_once(self, m_which): + net.openvswitch_is_installed() + net.openvswitch_is_installed() + assert 1 == m_which.call_count + + +@mock.patch("cloudinit.net.subp.subp", return_value=("", "")) +class TestGetOVSInternalInterfaces: + """Test cloudinit.net.get_ovs_internal_interfaces. + + Uses the ``clear_lru_cache`` local autouse fixture to allow us to test + despite the ``lru_cache`` decorator on the unit under test. + """ + @pytest.fixture(autouse=True) + def clear_lru_cache(self): + net.get_ovs_internal_interfaces.cache_clear() + + def test_command_used(self, m_subp): + """Test we use the correct command when we call subp""" + net.get_ovs_internal_interfaces() + + assert [ + mock.call(net.OVS_INTERNAL_INTERFACE_LOOKUP_CMD) + ] == m_subp.call_args_list + + def test_subp_contents_split_and_returned(self, m_subp): + """Test that the command output is appropriately mangled.""" + stdout = "iface1\niface2\niface3\n" + m_subp.return_value = (stdout, "") + + assert [ + "iface1", + "iface2", + "iface3", + ] == net.get_ovs_internal_interfaces() + + def test_database_connection_error_handled_gracefully(self, m_subp): + """Test that the error indicating OVS is down is handled gracefully.""" + m_subp.side_effect = ProcessExecutionError( + stderr="database connection failed" + ) + + assert [] == net.get_ovs_internal_interfaces() + + def test_other_errors_raised(self, m_subp): + """Test that only database connection errors are handled.""" + m_subp.side_effect = ProcessExecutionError() + + with pytest.raises(ProcessExecutionError): + net.get_ovs_internal_interfaces() + + def test_only_runs_once(self, m_subp): + """Test that we cache the value.""" + net.get_ovs_internal_interfaces() + net.get_ovs_internal_interfaces() + + assert 1 == m_subp.call_count + + +@mock.patch("cloudinit.net.get_ovs_internal_interfaces") +@mock.patch("cloudinit.net.openvswitch_is_installed") +class TestIsOpenVSwitchInternalInterface: + def test_false_if_ovs_not_installed( + self, m_openvswitch_is_installed, _m_get_ovs_internal_interfaces + ): + """Test that OVS' absence returns False.""" + m_openvswitch_is_installed.return_value = False + + assert not net.is_openvswitch_internal_interface("devname") + + @pytest.mark.parametrize( + "detected_interfaces,devname,expected_return", + [ + ([], "devname", False), + (["notdevname"], "devname", False), + (["devname"], "devname", True), + (["some", "other", "devices", "and", "ours"], "ours", True), + ], + ) + def test_return_value_based_on_detected_interfaces( + self, + m_openvswitch_is_installed, + m_get_ovs_internal_interfaces, + detected_interfaces, + devname, + expected_return, + ): + """Test that the detected interfaces are used correctly.""" + m_openvswitch_is_installed.return_value = True + m_get_ovs_internal_interfaces.return_value = detected_interfaces + assert expected_return == net.is_openvswitch_internal_interface( + devname + ) + + +class TestIsIpAddress: + """Tests for net.is_ip_address. + + Instead of testing with values we rely on the ipaddress stdlib module to + handle all values correctly, so simply test that is_ip_address defers to + the ipaddress module correctly. + """ + + @pytest.mark.parametrize('ip_address_side_effect,expected_return', ( + (ValueError, False), + (lambda _: ipaddress.IPv4Address('192.168.0.1'), True), + (lambda _: ipaddress.IPv6Address('2001:db8::'), True), + )) + def test_is_ip_address(self, ip_address_side_effect, expected_return): + with mock.patch('cloudinit.net.ipaddress.ip_address', + side_effect=ip_address_side_effect) as m_ip_address: + ret = net.is_ip_address(mock.sentinel.ip_address_in) + assert expected_return == ret + expected_call = mock.call(mock.sentinel.ip_address_in) + assert [expected_call] == m_ip_address.call_args_list + + +class TestIsIpv4Address: + """Tests for net.is_ipv4_address. + + Instead of testing with values we rely on the ipaddress stdlib module to + handle all values correctly, so simply test that is_ipv4_address defers to + the ipaddress module correctly. + """ + + @pytest.mark.parametrize('ipv4address_mock,expected_return', ( + (mock.Mock(side_effect=ValueError), False), + (mock.Mock(return_value=ipaddress.IPv4Address('192.168.0.1')), True), + )) + def test_is_ip_address(self, ipv4address_mock, expected_return): + with mock.patch('cloudinit.net.ipaddress.IPv4Address', + ipv4address_mock) as m_ipv4address: + ret = net.is_ipv4_address(mock.sentinel.ip_address_in) + assert expected_return == ret + expected_call = mock.call(mock.sentinel.ip_address_in) + assert [expected_call] == m_ipv4address.call_args_list + + # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/net/tests/test_network_state.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/tests/test_network_state.py --- cloud-init-20.1-10-g71af48df/cloudinit/net/tests/test_network_state.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/net/tests/test_network_state.py 2021-05-11 16:34:29.000000000 +0000 @@ -45,4 +45,14 @@ self.assertNotEqual(None, result) +class TestNetworkStateParseConfigV2(CiTestCase): + + def test_version_2_ignores_renderer_key(self): + ncfg = {'version': 2, 'renderer': 'networkd', 'ethernets': {}} + nsi = network_state.NetworkStateInterpreter(version=ncfg['version'], + config=ncfg) + nsi.parse_config(skip_broken=False) + self.assertEqual(ncfg, nsi.as_dict()['config']) + + # vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/netinfo.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/netinfo.py --- cloud-init-20.1-10-g71af48df/cloudinit/netinfo.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/netinfo.py 2021-05-11 16:34:29.000000000 +0000 @@ -13,6 +13,7 @@ from cloudinit import log as logging from cloudinit.net.network_state import net_prefix_to_ipv4_mask +from cloudinit import subp from cloudinit import util from cloudinit.simpletable import SimpleTable @@ -91,6 +92,53 @@ return devs +def _netdev_info_ifconfig_netbsd(ifconfig_data): + # fields that need to be returned in devs for each dev + devs = {} + for line in ifconfig_data.splitlines(): + if len(line) == 0: + continue + if line[0] not in ("\t", " "): + curdev = line.split()[0] + # current ifconfig pops a ':' on the end of the device + if curdev.endswith(':'): + curdev = curdev[:-1] + if curdev not in devs: + devs[curdev] = deepcopy(DEFAULT_NETDEV_INFO) + toks = line.lower().strip().split() + if len(toks) > 1: + if re.search(r"flags=[x\d]+", toks[1]): + devs[curdev]['up'] = True + + for i in range(len(toks)): + if toks[i] == "inet": # Create new ipv4 addr entry + network, net_bits = toks[i + 1].split('/') + devs[curdev]['ipv4'].append( + {'ip': network, 'mask': net_prefix_to_ipv4_mask(net_bits)}) + elif toks[i] == "broadcast": + devs[curdev]['ipv4'][-1]['bcast'] = toks[i + 1] + elif toks[i] == "address:": + devs[curdev]['hwaddr'] = toks[i + 1] + elif toks[i] == "inet6": + if toks[i + 1] == "addr:": + devs[curdev]['ipv6'].append({'ip': toks[i + 2]}) + else: + devs[curdev]['ipv6'].append({'ip': toks[i + 1]}) + elif toks[i] == "prefixlen": # Add prefix to current ipv6 value + addr6 = devs[curdev]['ipv6'][-1]['ip'] + "/" + toks[i + 1] + devs[curdev]['ipv6'][-1]['ip'] = addr6 + elif toks[i].startswith("scope:"): + devs[curdev]['ipv6'][-1]['scope6'] = toks[i].lstrip("scope:") + elif toks[i] == "scopeid": + res = re.match(r'.*<(\S+)>', toks[i + 1]) + if res: + devs[curdev]['ipv6'][-1]['scope6'] = res.group(1) + else: + devs[curdev]['ipv6'][-1]['scope6'] = toks[i + 1] + + return devs + + def _netdev_info_ifconfig(ifconfig_data): # fields that need to be returned in devs for each dev devs = {} @@ -149,13 +197,16 @@ def netdev_info(empty=""): devs = {} - if util.which('ip'): + if util.is_NetBSD(): + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) + devs = _netdev_info_ifconfig_netbsd(ifcfg_out) + elif subp.which('ip'): # Try iproute first of all - (ipaddr_out, _err) = util.subp(["ip", "addr", "show"]) + (ipaddr_out, _err) = subp.subp(["ip", "addr", "show"]) devs = _netdev_info_iproute(ipaddr_out) - elif util.which('ifconfig'): + elif subp.which('ifconfig'): # Fall back to net-tools if iproute2 is not present - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = _netdev_info_ifconfig(ifcfg_out) else: LOG.warning( @@ -235,10 +286,10 @@ entry['flags'] = ''.join(flags) routes['ipv4'].append(entry) try: - (iproute_data6, _err6) = util.subp( + (iproute_data6, _err6) = subp.subp( ["ip", "--oneline", "-6", "route", "list", "table", "all"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = iproute_data6.splitlines() @@ -307,9 +358,9 @@ routes['ipv4'].append(entry) try: - (route_data6, _err6) = util.subp( + (route_data6, _err6) = subp.subp( ["netstat", "-A", "inet6", "--route", "--numeric"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = route_data6.splitlines() @@ -343,13 +394,13 @@ def route_info(): routes = {} - if util.which('ip'): + if subp.which('ip'): # Try iproute first of all - (iproute_out, _err) = util.subp(["ip", "-o", "route", "list"]) + (iproute_out, _err) = subp.subp(["ip", "-o", "route", "list"]) routes = _netdev_route_info_iproute(iproute_out) - elif util.which('netstat'): + elif subp.which('netstat'): # Fall back to net-tools if iproute2 is not present - (route_out, _err) = util.subp( + (route_out, _err) = subp.subp( ["netstat", "--route", "--numeric", "--extend"], rcs=[0, 1]) routes = _netdev_route_info_netstat(route_out) else: diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/persistence.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/persistence.py --- cloud-init-20.1-10-g71af48df/cloudinit/persistence.py 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/persistence.py 2021-05-11 16:34:29.000000000 +0000 @@ -0,0 +1,67 @@ +# Copyright (C) 2020 Canonical Ltd. +# +# Author: Daniel Watkins +# +# This file is part of cloud-init. See LICENSE file for license information. + + +class CloudInitPickleMixin: + """Scaffolding for versioning of pickles. + + This class implements ``__getstate__`` and ``__setstate__`` to provide + lightweight versioning of the pickles that are generated for classes which + use it. Versioning is done at the class level. + + The current version of a class's pickle should be set in the class variable + ``_ci_pkl_version``, as an int. If not overriden, it will default to 0. + + On unpickle, the object's state will be restored and then + ``self._unpickle`` is called with the version of the stored pickle as the + only argument: this is where classes should implement any deserialization + fixes they require. (If the stored pickle has no version, 0 is passed.) + """ + + _ci_pkl_version = 0 + + def __getstate__(self): + """Persist instance state, adding a pickle version attribute. + + This adds a ``_ci_pkl_version`` attribute to ``self.__dict__`` and + returns that for serialisation. The attribute is stripped out in + ``__setstate__`` on unpickle. + + The value of ``_ci_pkl_version`` is ``type(self)._ci_pkl_version``. + """ + state = self.__dict__.copy() + state["_ci_pkl_version"] = type(self)._ci_pkl_version + return state + + def __setstate__(self, state: dict) -> None: + """Restore instance state and handle missing attributes on upgrade. + + This will be called when an instance of this class is unpickled; the + previous instance's ``__dict__`` is passed as ``state``. This method + removes the pickle version from the stored state, restores the + remaining state into the current instance, and then calls + ``self._unpickle`` with the version (or 0, if no version is found in + the stored state). + + See https://docs.python.org/3/library/pickle.html#object.__setstate__ + for further background. + """ + version = state.pop("_ci_pkl_version", 0) + self.__dict__.update(state) + self._unpickle(version) + + def _unpickle(self, ci_pkl_version: int) -> None: + """Perform any deserialization fixes required. + + By default, this does nothing. Classes using this mixin should + override this method if they have fixes they need to apply. + + ``ci_pkl_version`` will be the version stored in the pickle for this + object, or 0 if no version is present. + """ + + +# vi: ts=4 expandtab diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/reporting/events.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/reporting/events.py --- cloud-init-20.1-10-g71af48df/cloudinit/reporting/events.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/reporting/events.py 2021-05-11 16:34:29.000000000 +0000 @@ -12,7 +12,7 @@ import os.path import time -from . import instantiated_handler_registry +from . import instantiated_handler_registry, available_handlers FINISH_EVENT_TYPE = 'finish' START_EVENT_TYPE = 'start' @@ -81,17 +81,32 @@ return data -def report_event(event): - """Report an event to all registered event handlers. +def report_event(event, excluded_handler_types=None): + """Report an event to all registered event handlers + except those whose type is in excluded_handler_types. This should generally be called via one of the other functions in the reporting module. + :param excluded_handler_types: + List of handlers types to exclude from reporting the event to. :param event_type: The type of the event; this should be a constant from the reporting module. """ - for _, handler in instantiated_handler_registry.registered_items.items(): + + if not excluded_handler_types: + excluded_handler_types = {} + excluded_handler_classes = { + hndl_cls + for hndl_type, hndl_cls in available_handlers.registered_items.items() + if hndl_type in excluded_handler_types + } + + handlers = instantiated_handler_registry.registered_items.items() + for _, handler in handlers: + if type(handler) in excluded_handler_classes: + continue # skip this excluded handler handler.publish_event(event) diff -Nru cloud-init-20.1-10-g71af48df/cloudinit/reporting/handlers.py cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/reporting/handlers.py --- cloud-init-20.1-10-g71af48df/cloudinit/reporting/handlers.py 2020-03-10 14:22:22.000000000 +0000 +++ cloud-init-21.3-1-g6803368d+really.21.2-3-g899bfaa9/cloudinit/reporting/handlers.py 2021-05-11 16:34:29.000000000 +0000 @@ -35,7 +35,6 @@ def flush(self): """Ensure ReportingHandler has published all events""" - pass class LogHandler(ReportingHandler): @@ -114,6 +113,8 @@ https://technet.microsoft.com/en-us/library/dn798287.aspx#Linux%20guests """ HV_KVP_EXCHANGE_MAX_VALUE_SIZE = 2048 + # The maximum value size expected in Azure + HV_KVP_AZURE_MAX_VALUE_SIZE = 1024 HV_KVP_EXCHANGE_MAX_KEY_SIZE = 512 HV_KVP_RECORD_SIZE = (HV_KVP_EXCHANGE_MAX_KEY_SIZE + HV_KVP_EXCHANGE_MAX_VALUE_SIZE) @@ -139,7 +140,8 @@ self.event_key_prefix = u"{0}|{1}".format(self.EVENT_PREFIX, self.incarnation_no) self.publish_thread = threading.Thread( - target=self._publish_event_routine) + target=self._publish_event_routine + ) self.publish_thread.daemon = True self.publish_thread.start() @@ -195,17 +197,23 @@ def _event_key(self, event): """ the event key format is: - CLOUD_INIT||||