diff -Nru cloud-init-21.1-19-gbad84ad4/debian/changelog cloud-init-21.1-19-gbad84ad4/debian/changelog --- cloud-init-21.1-19-gbad84ad4/debian/changelog 2021-03-22 15:46:02.000000000 +0000 +++ cloud-init-21.1-19-gbad84ad4/debian/changelog 2021-04-19 19:02:27.000000000 +0000 @@ -1,3 +1,12 @@ +cloud-init (21.1-19-gbad84ad4-0ubuntu1~18.04.2) bionic; urgency=medium + + * cherry-pick 83f6bbfb: Fix unpickle for source paths missing run_dir + (#863) (LP: #1899299) + * cherry-pick d132356c: fix error on upgrade caused by new vendordata2 + attributes (LP: #1922739) + + -- James Falcon Mon, 19 Apr 2021 14:02:27 -0500 + cloud-init (21.1-19-gbad84ad4-0ubuntu1~18.04.1) bionic; urgency=medium * d/cloud-init.postinst: Change output log permissions on upgrade (LP: #1918303) diff -Nru cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863 cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863 --- cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863 2021-04-19 19:02:27.000000000 +0000 @@ -0,0 +1,576 @@ +From 83f6bbfbe5b924be61a3c098f4202377d69c8947 Mon Sep 17 00:00:00 2001 +From: lucasmoura +Date: Mon, 12 Apr 2021 13:22:22 -0300 +Subject: [PATCH] Fix unpickle for source paths missing run_dir (#863) + +On the datasource class, we require the use of paths.run_dir to +perform some operations. On older cloud-init version, the +Paths class does not have the run_dir attribute. To fix that, +we are now manually adding that attribute in the Paths +object if doesn't exist in the unpickle operation. + +LP: #1899299 +--- + cloudinit/helpers.py | 17 +- + cloudinit/tests/test_upgrade.py | 3 + + .../data/old_pickles/trusty-14.04.1-0.7.5.pkl | 504 ++++++++++++++++++ + 3 files changed, 523 insertions(+), 1 deletion(-) + create mode 100644 tests/data/old_pickles/trusty-14.04.1-0.7.5.pkl + +--- a/cloudinit/helpers.py ++++ b/cloudinit/helpers.py +@@ -20,6 +20,7 @@ from cloudinit.settings import (PER_INST + + from cloudinit import log as logging + from cloudinit import type_utils ++from cloudinit import persistence + from cloudinit import util + + LOG = logging.getLogger(__name__) +@@ -317,7 +318,9 @@ class ContentHandlers(object): + return list(self.registered.items()) + + +-class Paths(object): ++class Paths(persistence.CloudInitPickleMixin): ++ _ci_pkl_version = 1 ++ + def __init__(self, path_cfgs, ds=None): + self.cfgs = path_cfgs + # Populate all the initial paths +@@ -354,6 +357,18 @@ class Paths(object): + # Set when a datasource becomes active + self.datasource = ds + ++ def _unpickle(self, ci_pkl_version: int) -> None: ++ """Perform deserialization fixes for Paths.""" ++ if not hasattr(self, "run_dir"): ++ # On older versions of cloud-init the Paths class do not ++ # have the run_dir attribute. This is problematic because ++ # when loading the pickle object on newer versions of cloud-init ++ # we will rely on this attribute. To fix that, we are now ++ # manually adding that attribute here. ++ self.run_dir = Paths( ++ path_cfgs=self.cfgs, ++ ds=self.datasource).run_dir ++ + # get_ipath_cur: get the current instance path for an item + def get_ipath_cur(self, name=None): + return self._get_path(self.instance_link, name) +--- a/cloudinit/tests/test_upgrade.py ++++ b/cloudinit/tests/test_upgrade.py +@@ -43,3 +43,6 @@ class TestUpgrade: + def test_blacklist_drivers_set_on_networking(self, previous_obj_pkl): + """We always expect Networking.blacklist_drivers to be initialised.""" + assert previous_obj_pkl.distro.networking.blacklist_drivers is None ++ ++ def test_paths_has_run_dir_attribute(self, previous_obj_pkl): ++ assert previous_obj_pkl.paths.run_dir is not None +--- /dev/null ++++ b/tests/data/old_pickles/trusty-14.04.1-0.7.5.pkl +@@ -0,0 +1,504 @@ ++ccopy_reg ++_reconstructor ++p1 ++(ccloudinit.sources.DataSourceNoCloud ++DataSourceNoCloudNet ++p2 ++c__builtin__ ++object ++p3 ++NtRp4 ++(dp5 ++S'paths' ++p6 ++g1 ++(ccloudinit.helpers ++Paths ++p7 ++g3 ++NtRp8 ++(dp9 ++S'lookups' ++p10 ++(dp11 ++S'cloud_config' ++p12 ++S'cloud-config.txt' ++p13 ++sS'userdata' ++p14 ++S'user-data.txt.i' ++p15 ++sS'vendordata' ++p16 ++S'vendor-data.txt.i' ++p17 ++sS'userdata_raw' ++p18 ++S'user-data.txt' ++p19 ++sS'boothooks' ++p20 ++g20 ++sS'scripts' ++p21 ++g21 ++sS'sem' ++p22 ++g22 ++sS'data' ++p23 ++g23 ++sS'vendor_scripts' ++p24 ++S'scripts/vendor' ++p25 ++sS'handlers' ++p26 ++g26 ++sS'obj_pkl' ++p27 ++S'obj.pkl' ++p28 ++sS'vendordata_raw' ++p29 ++S'vendor-data.txt' ++p30 ++sS'vendor_cloud_config' ++p31 ++S'vendor-cloud-config.txt' ++p32 ++ssS'template_tpl' ++p33 ++S'/etc/cloud/templates/%s.tmpl' ++p34 ++sS'cfgs' ++p35 ++(dp36 ++S'cloud_dir' ++p37 ++S'/var/lib/cloud/' ++p38 ++sS'templates_dir' ++p39 ++S'/etc/cloud/templates/' ++p40 ++sS'upstart_dir' ++p41 ++S'/etc/init/' ++p42 ++ssS'cloud_dir' ++p43 ++g38 ++sS'datasource' ++p44 ++NsS'upstart_conf_d' ++p45 ++g42 ++sS'boot_finished' ++p46 ++S'/var/lib/cloud/instance/boot-finished' ++p47 ++sS'instance_link' ++p48 ++S'/var/lib/cloud/instance' ++p49 ++sS'seed_dir' ++p50 ++S'/var/lib/cloud/seed' ++p51 ++sbsS'supported_seed_starts' ++p52 ++(S'http://' ++S'https://' ++S'ftp://' ++tp53 ++sS'sys_cfg' ++p54 ++(dp55 ++S'output' ++p56 ++(dp57 ++S'all' ++p58 ++S'| tee -a /var/log/cloud-init-output.log' ++p59 ++ssS'users' ++p60 ++(lp61 ++S'default' ++p62 ++asS'def_log_file' ++p63 ++S'/var/log/cloud-init.log' ++p64 ++sS'cloud_final_modules' ++p65 ++(lp66 ++S'rightscale_userdata' ++p67 ++aS'scripts-vendor' ++p68 ++aS'scripts-per-once' ++p69 ++aS'scripts-per-boot' ++p70 ++aS'scripts-per-instance' ++p71 ++aS'scripts-user' ++p72 ++aS'ssh-authkey-fingerprints' ++p73 ++aS'keys-to-console' ++p74 ++aS'phone-home' ++p75 ++aS'final-message' ++p76 ++aS'power-state-change' ++p77 ++asS'disable_root' ++p78 ++I01 ++sS'syslog_fix_perms' ++p79 ++S'syslog:adm' ++p80 ++sS'log_cfgs' ++p81 ++(lp82 ++(lp83 ++S'[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n' ++p84 ++aS'[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=("/dev/log", handlers.SysLogHandler.LOG_USER)\n' ++p85 ++aa(lp86 ++g84 ++aS"[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n" ++p87 ++aasS'cloud_init_modules' ++p88 ++(lp89 ++S'migrator' ++p90 ++aS'seed_random' ++p91 ++aS'bootcmd' ++p92 ++aS'write-files' ++p93 ++aS'growpart' ++p94 ++aS'resizefs' ++p95 ++aS'set_hostname' ++p96 ++aS'update_hostname' ++p97 ++aS'update_etc_hosts' ++p98 ++aS'ca-certs' ++p99 ++aS'rsyslog' ++p100 ++aS'users-groups' ++p101 ++aS'ssh' ++p102 ++asS'preserve_hostname' ++p103 ++I00 ++sS'_log' ++p104 ++(lp105 ++g84 ++ag87 ++ag85 ++asS'datasource_list' ++p106 ++(lp107 ++S'NoCloud' ++p108 ++aS'ConfigDrive' ++p109 ++aS'OpenNebula' ++p110 ++aS'Azure' ++p111 ++aS'AltCloud' ++p112 ++aS'OVF' ++p113 ++aS'MAAS' ++p114 ++aS'GCE' ++p115 ++aS'OpenStack' ++p116 ++aS'CloudSigma' ++p117 ++aS'Ec2' ++p118 ++aS'CloudStack' ++p119 ++aS'SmartOS' ++p120 ++aS'None' ++p121 ++asS'vendor_data' ++p122 ++(dp123 ++S'prefix' ++p124 ++(lp125 ++sS'enabled' ++p126 ++I01 ++ssS'cloud_config_modules' ++p127 ++(lp128 ++S'emit_upstart' ++p129 ++aS'disk_setup' ++p130 ++aS'mounts' ++p131 ++aS'ssh-import-id' ++p132 ++aS'locale' ++p133 ++aS'set-passwords' ++p134 ++aS'grub-dpkg' ++p135 ++aS'apt-pipelining' ++p136 ++aS'apt-configure' ++p137 ++aS'package-update-upgrade-install' ++p138 ++aS'landscape' ++p139 ++aS'timezone' ++p140 ++aS'puppet' ++p141 ++aS'chef' ++p142 ++aS'salt-minion' ++p143 ++aS'mcollective' ++p144 ++aS'disable-ec2-metadata' ++p145 ++aS'runcmd' ++p146 ++aS'byobu' ++p147 ++assg14 ++Nsg16 ++Nsg18 ++S'#cloud-config\n{}\n\n' ++p148 ++sg29 ++S'#cloud-config\n{}\n\n' ++p149 ++sS'dsmode' ++p150 ++S'net' ++p151 ++sS'seed' ++p152 ++S'/var/lib/cloud/seed/nocloud-net' ++p153 ++sS'cmdline_id' ++p154 ++S'ds=nocloud-net' ++p155 ++sS'ud_proc' ++p156 ++g1 ++(ccloudinit.user_data ++UserDataProcessor ++p157 ++g3 ++NtRp158 ++(dp159 ++g6 ++g8 ++sS'ssl_details' ++p160 ++(dp161 ++sbsg50 ++g153 ++sS'ds_cfg' ++p162 ++(dp163 ++sS'distro' ++p164 ++g1 ++(ccloudinit.distros.ubuntu ++Distro ++p165 ++g3 ++NtRp166 ++(dp167 ++S'osfamily' ++p168 ++S'debian' ++p169 ++sS'_paths' ++p170 ++g8 ++sS'name' ++p171 ++S'ubuntu' ++p172 ++sS'_runner' ++p173 ++g1 ++(ccloudinit.helpers ++Runners ++p174 ++g3 ++NtRp175 ++(dp176 ++g6 ++g8 ++sS'sems' ++p177 ++(dp178 ++sbsS'_cfg' ++p179 ++(dp180 ++S'paths' ++p181 ++(dp182 ++g37 ++g38 ++sg39 ++g40 ++sg41 ++g42 ++ssS'default_user' ++p183 ++(dp184 ++S'shell' ++p185 ++S'/bin/bash' ++p186 ++sS'name' ++p187 ++S'ubuntu' ++p188 ++sS'sudo' ++p189 ++(lp190 ++S'ALL=(ALL) NOPASSWD:ALL' ++p191 ++asS'lock_passwd' ++p192 ++I01 ++sS'gecos' ++p193 ++S'Ubuntu' ++p194 ++sS'groups' ++p195 ++(lp196 ++S'adm' ++p197 ++aS'audio' ++p198 ++aS'cdrom' ++p199 ++aS'dialout' ++p200 ++aS'dip' ++p201 ++aS'floppy' ++p202 ++aS'netdev' ++p203 ++aS'plugdev' ++p204 ++aS'sudo' ++p205 ++aS'video' ++p206 ++assS'package_mirrors' ++p207 ++(lp208 ++(dp209 ++S'arches' ++p210 ++(lp211 ++S'i386' ++p212 ++aS'amd64' ++p213 ++asS'failsafe' ++p214 ++(dp215 ++S'security' ++p216 ++S'http://security.ubuntu.com/ubuntu' ++p217 ++sS'primary' ++p218 ++S'http://archive.ubuntu.com/ubuntu' ++p219 ++ssS'search' ++p220 ++(dp221 ++S'security' ++p222 ++(lp223 ++sS'primary' ++p224 ++(lp225 ++S'http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/' ++p226 ++aS'http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/' ++p227 ++aS'http://%(region)s.clouds.archive.ubuntu.com/ubuntu/' ++p228 ++assa(dp229 ++S'arches' ++p230 ++(lp231 ++S'armhf' ++p232 ++aS'armel' ++p233 ++aS'default' ++p234 ++asS'failsafe' ++p235 ++(dp236 ++S'security' ++p237 ++S'http://ports.ubuntu.com/ubuntu-ports' ++p238 ++sS'primary' ++p239 ++S'http://ports.ubuntu.com/ubuntu-ports' ++p240 ++ssasS'ssh_svcname' ++p241 ++S'ssh' ++p242 ++ssbsS'metadata' ++p243 ++(dp244 ++g150 ++g151 ++sS'local-hostname' ++p245 ++S'trusty-upgrade2' ++p246 ++sS'instance-id' ++p247 ++S'trusty-upgrade2' ++p248 ++ssb. +\ No newline at end of file diff -Nru cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2 cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2 --- cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2 1970-01-01 00:00:00.000000000 +0000 +++ cloud-init-21.1-19-gbad84ad4/debian/patches/cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2 2021-04-19 19:02:27.000000000 +0000 @@ -0,0 +1,147 @@ +From d132356cc361abef2d90d4073438f3ab759d5964 Mon Sep 17 00:00:00 2001 +From: James Falcon +Date: Mon, 19 Apr 2021 11:31:28 -0500 +Subject: [PATCH] fix error on upgrade caused by new vendordata2 attributes + (#869) + +In #777, we added 'vendordata2' and 'vendordata2_raw' attributes to +the DataSource class, but didn't use the upgrade framework to deal +with an unpickle after upgrade. This commit adds the necessary +upgrade code. + +Additionally, added a smaller-scope upgrade test to our integration +tests that will be run on every CI run so we catch these issues +immediately in the future. + +LP: #1922739 +--- + cloudinit/sources/__init__.py | 12 +++++++++++- + cloudinit/tests/test_upgrade.py | 4 ++++ + tests/integration_tests/clouds.py | 4 ++-- + tests/integration_tests/test_upgrade.py | 25 ++++++++++++++++++++++++- + 4 files changed, 41 insertions(+), 4 deletions(-) + +--- a/cloudinit/sources/__init__.py ++++ b/cloudinit/sources/__init__.py +@@ -24,6 +24,7 @@ from cloudinit import util + from cloudinit.atomic_helper import write_json + from cloudinit.event import EventType + from cloudinit.filters import launch_index ++from cloudinit.persistence import CloudInitPickleMixin + from cloudinit.reporting import events + + DSMODE_DISABLED = "disabled" +@@ -134,7 +135,7 @@ URLParams = namedtuple( + 'URLParms', ['max_wait_seconds', 'timeout_seconds', 'num_retries']) + + +-class DataSource(metaclass=abc.ABCMeta): ++class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): + + dsmode = DSMODE_NETWORK + default_locale = 'en_US.UTF-8' +@@ -196,6 +197,8 @@ class DataSource(metaclass=abc.ABCMeta): + # non-root users + sensitive_metadata_keys = ('merged_cfg', 'security-credentials',) + ++ _ci_pkl_version = 1 ++ + def __init__(self, sys_cfg, distro, paths, ud_proc=None): + self.sys_cfg = sys_cfg + self.distro = distro +@@ -218,6 +221,13 @@ class DataSource(metaclass=abc.ABCMeta): + else: + self.ud_proc = ud_proc + ++ def _unpickle(self, ci_pkl_version: int) -> None: ++ """Perform deserialization fixes for Paths.""" ++ if not hasattr(self, 'vendordata2'): ++ self.vendordata2 = None ++ if not hasattr(self, 'vendordata2_raw'): ++ self.vendordata2_raw = None ++ + def __str__(self): + return type_utils.obj_name(self) + +--- a/cloudinit/tests/test_upgrade.py ++++ b/cloudinit/tests/test_upgrade.py +@@ -46,3 +46,7 @@ class TestUpgrade: + + def test_paths_has_run_dir_attribute(self, previous_obj_pkl): + assert previous_obj_pkl.paths.run_dir is not None ++ ++ def test_vendordata_exists(self, previous_obj_pkl): ++ assert previous_obj_pkl.vendordata2 is None ++ assert previous_obj_pkl.vendordata2_raw is None +--- a/tests/integration_tests/clouds.py ++++ b/tests/integration_tests/clouds.py +@@ -110,14 +110,14 @@ class IntegrationCloud(ABC): + # Even if we're using the default key, it may still have a + # different name in the clouds, so we need to set it separately. + self.cloud_instance.key_pair.name = settings.KEYPAIR_NAME +- self._released_image_id = self._get_initial_image() ++ self.released_image_id = self._get_initial_image() + self.snapshot_id = None + + @property + def image_id(self): + if self.snapshot_id: + return self.snapshot_id +- return self._released_image_id ++ return self.released_image_id + + def emit_settings_to_log(self) -> None: + log.info( +--- a/tests/integration_tests/test_upgrade.py ++++ b/tests/integration_tests/test_upgrade.py +@@ -1,4 +1,5 @@ + import logging ++import os + import pytest + import time + from pathlib import Path +@@ -8,6 +9,8 @@ from tests.integration_tests.conftest im + get_validated_source, + session_start_time, + ) ++from tests.integration_tests.instances import CloudInitSource ++ + + log = logging.getLogger('integration_testing') + +@@ -63,7 +66,7 @@ def test_upgrade(session_cloud: Integrat + return # type checking doesn't understand that skip raises + + launch_kwargs = { +- 'image_id': session_cloud._get_initial_image(), ++ 'image_id': session_cloud.released_image_id, + } + + image = ImageSpecification.from_os_image() +@@ -93,6 +96,26 @@ def test_upgrade(session_cloud: Integrat + instance.install_new_cloud_init(source, take_snapshot=False) + instance.execute('hostname something-else') + _restart(instance) ++ assert instance.execute('cloud-init status --wait --long').ok + _output_to_compare(instance, after_path, netcfg_path) + + log.info('Wrote upgrade test logs to %s and %s', before_path, after_path) ++ ++ ++@pytest.mark.ci ++@pytest.mark.ubuntu ++def test_upgrade_package(session_cloud: IntegrationCloud): ++ if get_validated_source(session_cloud) != CloudInitSource.DEB_PACKAGE: ++ not_run_message = 'Test only supports upgrading to build deb' ++ if os.environ.get('TRAVIS'): ++ # If this isn't running on CI, we should know ++ pytest.fail(not_run_message) ++ else: ++ pytest.skip(not_run_message) ++ ++ launch_kwargs = {'image_id': session_cloud.released_image_id} ++ ++ with session_cloud.launch(launch_kwargs=launch_kwargs) as instance: ++ instance.install_deb() ++ instance.restart() ++ assert instance.execute('cloud-init status --wait --long').ok diff -Nru cloud-init-21.1-19-gbad84ad4/debian/patches/series cloud-init-21.1-19-gbad84ad4/debian/patches/series --- cloud-init-21.1-19-gbad84ad4/debian/patches/series 2021-03-22 15:46:02.000000000 +0000 +++ cloud-init-21.1-19-gbad84ad4/debian/patches/series 2021-04-19 19:02:27.000000000 +0000 @@ -2,3 +2,5 @@ ubuntu-advantage-revert-tip.patch ec2-dont-apply-full-imds-network-config.patch renderer-do-not-prefer-netplan.patch +cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863 +cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2