diff -Nru cloud-init-23.4.2/ChangeLog cloud-init-23.4.3/ChangeLog --- cloud-init-23.4.2/ChangeLog 2024-01-24 17:06:27.000000000 +0000 +++ cloud-init-23.4.3/ChangeLog 2024-02-02 21:21:52.000000000 +0000 @@ -1,3 +1,7 @@ +23.4.3 + - fix: Handle systemctl when dbus not ready (#4842) + (LP: #2046483) + 23.4.2 - fix: Handle invalid user configuration gracefully (#4797) (LP: #2051147) diff -Nru cloud-init-23.4.2/cloudinit/cmd/status.py cloud-init-23.4.3/cloudinit/cmd/status.py --- cloud-init-23.4.2/cloudinit/cmd/status.py 2024-01-24 17:06:27.000000000 +0000 +++ cloud-init-23.4.3/cloudinit/cmd/status.py 2024-02-02 21:21:52.000000000 +0000 @@ -93,6 +93,36 @@ {description}""" +def query_systemctl( + systemctl_args: List[str], + *, + wait: bool, + existing_status: Optional[UXAppStatus] = None, +) -> str: + """Query systemd with retries and return output.""" + while True: + try: + return subp.subp(["systemctl", *systemctl_args]).stdout.strip() + except subp.ProcessExecutionError as e: + if existing_status and existing_status in ( + UXAppStatus.DEGRADED_RUNNING, + UXAppStatus.RUNNING, + ): + return "" + last_exception = e + if wait: + sleep(0.25) + else: + break + print( + "Failed to get status from systemd. " + "Cloud-init status may be inaccurate. ", + f"Error from systemctl: {last_exception.stderr}", + file=sys.stderr, + ) + return "" + + def get_parser(parser=None): """Build or extend an arg parser for status utility. @@ -229,7 +259,9 @@ return 0 -def get_bootstatus(disable_file, paths) -> Tuple[UXAppBootStatusCode, str]: +def get_bootstatus( + disable_file, paths, wait +) -> Tuple[UXAppBootStatusCode, str]: """Report whether cloud-init current boot status @param disable_file: The path to the cloud-init disable file. @@ -253,7 +285,7 @@ elif "cloud-init=disabled" in os.environ.get("KERNEL_CMDLINE", "") or ( uses_systemd() and "cloud-init=disabled" - in subp.subp(["systemctl", "show-environment"]).stdout + in query_systemctl(["show-environment"], wait=wait) ): bootstatus_code = UXAppBootStatusCode.DISABLED_BY_ENV_VARIABLE reason = ( @@ -272,7 +304,9 @@ return (bootstatus_code, reason) -def _get_error_or_running_from_systemd() -> Optional[UXAppStatus]: +def _get_error_or_running_from_systemd( + existing_status: UXAppStatus, wait: bool +) -> Optional[UXAppStatus]: """Get if systemd is in error or running state. Using systemd, we can get more fine-grained status of the @@ -288,14 +322,18 @@ "cloud-init.service", "cloud-init-local.service", ]: - stdout = subp.subp( + stdout = query_systemctl( [ - "systemctl", "show", "--property=ActiveState,UnitFileState,SubState,MainPID", service, ], - ).stdout + wait=wait, + existing_status=existing_status, + ) + if not stdout: + # Systemd isn't ready + return None states = dict( [[x.strip() for x in r.split("=")] for r in stdout.splitlines()] ) @@ -327,39 +365,6 @@ return None -def _get_error_or_running_from_systemd_with_retry( - existing_status: UXAppStatus, *, wait: bool -) -> Optional[UXAppStatus]: - """Get systemd status and retry if dbus isn't ready. - - If cloud-init has determined that we're still running, then we can - ignore the status from systemd. However, if cloud-init has detected error, - then we should retry on systemd status so we don't incorrectly report - error state while cloud-init is still running. - """ - while True: - try: - return _get_error_or_running_from_systemd() - except subp.ProcessExecutionError as e: - last_exception = e - if existing_status in ( - UXAppStatus.DEGRADED_RUNNING, - UXAppStatus.RUNNING, - ): - return None - if wait: - sleep(0.25) - else: - break - print( - "Failed to get status from systemd. " - "Cloud-init status may be inaccurate. ", - f"Error from systemctl: {last_exception.stderr}", - file=sys.stderr, - ) - return None - - def get_status_details( paths: Optional[Paths] = None, wait: bool = False ) -> StatusDetails: @@ -380,7 +385,7 @@ result_file = os.path.join(paths.run_dir, "result.json") boot_status_code, description = get_bootstatus( - CLOUDINIT_DISABLED_FILE, paths + CLOUDINIT_DISABLED_FILE, paths, wait ) if boot_status_code in DISABLED_BOOT_CODES: status = UXAppStatus.DISABLED @@ -433,9 +438,7 @@ UXAppStatus.NOT_RUN, UXAppStatus.DISABLED, ): - systemd_status = _get_error_or_running_from_systemd_with_retry( - status, wait=wait - ) + systemd_status = _get_error_or_running_from_systemd(status, wait=wait) if systemd_status: status = systemd_status diff -Nru cloud-init-23.4.2/cloudinit/version.py cloud-init-23.4.3/cloudinit/version.py --- cloud-init-23.4.2/cloudinit/version.py 2024-01-24 17:06:27.000000000 +0000 +++ cloud-init-23.4.3/cloudinit/version.py 2024-02-02 21:21:52.000000000 +0000 @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "23.4.2" +__VERSION__ = "23.4.3" _PACKAGED_VERSION = "@@PACKAGED_VERSION@@" FEATURES = [ diff -Nru cloud-init-23.4.2/debian/changelog cloud-init-23.4.3/debian/changelog --- cloud-init-23.4.2/debian/changelog 2024-01-24 17:57:50.000000000 +0000 +++ cloud-init-23.4.3/debian/changelog 2024-02-02 22:00:04.000000000 +0000 @@ -1,3 +1,11 @@ +cloud-init (23.4.3-0ubuntu0~23.10.1) mantic; urgency=medium + + * Upstream snapshot based on 23.4.3. (LP: #2046483). + List of changes from upstream can be found at + https://raw.githubusercontent.com/canonical/cloud-init/23.4.3/ChangeLog + + -- James Falcon Fri, 02 Feb 2024 16:00:04 -0600 + cloud-init (23.4.2-0ubuntu0~23.10.1) mantic; urgency=medium * Upstream snapshot based on 23.4.2. (LP: #2045582). diff -Nru cloud-init-23.4.2/debian/patches/status-do-not-remove-duplicated-data.patch cloud-init-23.4.3/debian/patches/status-do-not-remove-duplicated-data.patch --- cloud-init-23.4.2/debian/patches/status-do-not-remove-duplicated-data.patch 2024-01-24 17:57:50.000000000 +0000 +++ cloud-init-23.4.3/debian/patches/status-do-not-remove-duplicated-data.patch 2024-02-02 22:00:04.000000000 +0000 @@ -7,7 +7,7 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/cloudinit/cmd/status.py +++ b/cloudinit/cmd/status.py -@@ -162,6 +162,8 @@ def handle_status_args(name, args) -> in +@@ -192,6 +192,8 @@ def handle_status_args(name, args) -> in "last_update": details.last_update, **details.v1, } @@ -18,7 +18,7 @@ prefix = "\n" if args.wait else "" --- a/tests/unittests/cmd/test_status.py +++ b/tests/unittests/cmd/test_status.py -@@ -487,6 +487,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -482,6 +482,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr dedent( """\ --- @@ -26,7 +26,7 @@ boot_status_code: enabled-by-kernel-cmdline datasource: '' detail: 'Running in stage: init' -@@ -500,6 +501,23 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -495,6 +496,23 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr start: 123.45 last_update: Thu, 01 Jan 1970 00:02:04 +0000 recoverable_errors: {} @@ -50,7 +50,7 @@ stage: init status: running ... -@@ -532,6 +550,25 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -527,6 +545,25 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr "init-local": {"finished": 123.46, "start": 123.45}, "last_update": "Thu, 01 Jan 1970 00:02:04 +0000", "recoverable_errors": {}, @@ -76,7 +76,7 @@ "stage": "init", }, id="running_json_format", -@@ -563,6 +600,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -558,6 +595,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr MyArgs(long=False, wait=False, format="json"), 1, { @@ -84,7 +84,7 @@ "boot_status_code": "enabled-by-kernel-cmdline", "datasource": "nocloud", "detail": ( -@@ -584,6 +622,32 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -579,6 +617,32 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr }, "last_update": "Thu, 01 Jan 1970 00:02:05 +0000", "recoverable_errors": {}, @@ -117,7 +117,7 @@ "stage": None, }, id="running_json_format_with_errors", -@@ -646,6 +710,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -641,6 +705,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr MyArgs(long=False, wait=False, format="json"), 2, { @@ -125,7 +125,7 @@ "boot_status_code": "enabled-by-kernel-cmdline", "datasource": "nocloud", "detail": ( -@@ -705,6 +770,89 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -700,6 +765,89 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr "don't try to open the hatch or we'll all be soup" ], }, diff -Nru cloud-init-23.4.2/debian/patches/status-retain-recoverable-error-exit-code.patch cloud-init-23.4.3/debian/patches/status-retain-recoverable-error-exit-code.patch --- cloud-init-23.4.2/debian/patches/status-retain-recoverable-error-exit-code.patch 2024-01-24 17:57:50.000000000 +0000 +++ cloud-init-23.4.3/debian/patches/status-retain-recoverable-error-exit-code.patch 2024-02-02 22:00:04.000000000 +0000 @@ -6,7 +6,7 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/cloudinit/cmd/status.py +++ b/cloudinit/cmd/status.py -@@ -227,7 +227,7 @@ def handle_status_args(name, args) -> in +@@ -257,7 +257,7 @@ def handle_status_args(name, args) -> in return 1 # Recoverable error elif details.status in UXAppStatusDegradedMap.values(): @@ -17,7 +17,7 @@ --- a/tests/unittests/cmd/test_status.py +++ b/tests/unittests/cmd/test_status.py -@@ -708,7 +708,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr +@@ -703,7 +703,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr }, None, MyArgs(long=False, wait=False, format="json"), diff -Nru cloud-init-23.4.2/tests/unittests/cmd/test_status.py cloud-init-23.4.3/tests/unittests/cmd/test_status.py --- cloud-init-23.4.2/tests/unittests/cmd/test_status.py 2024-01-24 17:06:27.000000000 +0000 +++ cloud-init-23.4.3/tests/unittests/cmd/test_status.py 2024-02-02 21:21:52.000000000 +0000 @@ -15,7 +15,6 @@ from cloudinit.cmd.status import ( UXAppStatus, _get_error_or_running_from_systemd, - _get_error_or_running_from_systemd_with_retry, ) from cloudinit.subp import SubpResult from cloudinit.util import ensure_file @@ -65,7 +64,7 @@ ), ) @mock.patch( - f"{M_PATH}_get_error_or_running_from_systemd_with_retry", + f"{M_PATH}_get_error_or_running_from_systemd", return_value=None, ) def test_get_status_details_ds_none( @@ -188,6 +187,7 @@ failure_msg: str, expected_reason: Union[str, Callable], config: Config, + mocker, ): if ensured_file is not None: ensure_file(ensured_file(config)) @@ -201,15 +201,10 @@ stderr=None, ), ): - code, reason = wrap_and_call( - M_NAME, - { - "uses_systemd": uses_systemd, - "get_cmdline": get_cmdline, - }, - status.get_bootstatus, - config.disable_file, - config.paths, + mocker.patch(f"{M_PATH}uses_systemd", return_value=uses_systemd) + mocker.patch(f"{M_PATH}get_cmdline", return_value=get_cmdline) + code, reason = status.get_bootstatus( + config.disable_file, config.paths, False ) assert code == expected_bootstatus, failure_msg if isinstance(expected_reason, str): @@ -713,7 +708,7 @@ ) @mock.patch(M_PATH + "read_cfg_paths") @mock.patch( - f"{M_PATH}_get_error_or_running_from_systemd_with_retry", + f"{M_PATH}_get_error_or_running_from_systemd", return_value=None, ) def test_status_output( @@ -757,7 +752,7 @@ @mock.patch(M_PATH + "read_cfg_paths") @mock.patch( - f"{M_PATH}_get_error_or_running_from_systemd_with_retry", + f"{M_PATH}_get_error_or_running_from_systemd", return_value=None, ) def test_status_wait_blocks_until_done( @@ -811,7 +806,7 @@ @mock.patch(M_PATH + "read_cfg_paths") @mock.patch( - f"{M_PATH}_get_error_or_running_from_systemd_with_retry", + f"{M_PATH}_get_error_or_running_from_systemd", return_value=None, ) def test_status_wait_blocks_until_error( @@ -867,7 +862,7 @@ @mock.patch(M_PATH + "read_cfg_paths") @mock.patch( - f"{M_PATH}_get_error_or_running_from_systemd_with_retry", + f"{M_PATH}_get_error_or_running_from_systemd", return_value=None, ) def test_status_main( @@ -948,7 +943,10 @@ stderr=None, ), ): - assert _get_error_or_running_from_systemd() == status + assert ( + _get_error_or_running_from_systemd(UXAppStatus.RUNNING, False) + == status + ) def test_exception_while_running(self, mocker, capsys): m_subp = mocker.patch( @@ -959,9 +957,7 @@ ), ) assert ( - _get_error_or_running_from_systemd_with_retry( - UXAppStatus.RUNNING, wait=True - ) + _get_error_or_running_from_systemd(UXAppStatus.RUNNING, wait=True) is None ) assert 1 == m_subp.call_count @@ -987,9 +983,7 @@ ], ) assert ( - _get_error_or_running_from_systemd_with_retry( - UXAppStatus.ERROR, wait=True - ) + _get_error_or_running_from_systemd(UXAppStatus.ERROR, wait=True) is UXAppStatus.RUNNING ) assert 3 == m_subp.call_count @@ -1005,9 +999,7 @@ ) mocker.patch("time.time", side_effect=[1, 2, 50]) assert ( - _get_error_or_running_from_systemd_with_retry( - UXAppStatus.ERROR, wait=False - ) + _get_error_or_running_from_systemd(UXAppStatus.ERROR, wait=False) is None ) assert 1 == m_subp.call_count @@ -1015,3 +1007,60 @@ "Failed to get status from systemd. " "Cloud-init status may be inaccurate." ) in capsys.readouterr().err + + +class TestQuerySystemctl: + def test_query_systemctl(self, mocker): + m_subp = mocker.patch( + f"{M_PATH}subp.subp", + return_value=SubpResult(stdout="hello", stderr=None), + ) + assert status.query_systemctl(["some", "args"], wait=False) == "hello" + m_subp.assert_called_once_with(["systemctl", "some", "args"]) + + def test_query_systemctl_with_exception(self, mocker, capsys): + m_subp = mocker.patch( + f"{M_PATH}subp.subp", + side_effect=subp.ProcessExecutionError( + "Message recipient disconnected", stderr="oh noes!" + ), + ) + assert status.query_systemctl(["some", "args"], wait=False) == "" + m_subp.assert_called_once_with(["systemctl", "some", "args"]) + assert "Error from systemctl: oh noes!" in capsys.readouterr().err + + def test_query_systemctl_wait_with_exception(self, mocker): + m_sleep = mocker.patch(f"{M_PATH}sleep") + m_subp = mocker.patch( + f"{M_PATH}subp.subp", + side_effect=[ + subp.ProcessExecutionError("Message recipient disconnected"), + subp.ProcessExecutionError("Message recipient disconnected"), + subp.ProcessExecutionError("Message recipient disconnected"), + SubpResult(stdout="hello", stderr=None), + ], + ) + + assert status.query_systemctl(["some", "args"], wait=True) == "hello" + assert m_subp.call_count == 4 + assert m_sleep.call_count == 3 + + def test_query_systemctl_wait_with_exception_status(self, mocker): + m_sleep = mocker.patch(f"{M_PATH}sleep") + m_subp = mocker.patch( + f"{M_PATH}subp.subp", + side_effect=subp.ProcessExecutionError( + "Message recipient disconnected" + ), + ) + + assert ( + status.query_systemctl( + ["some", "args"], + wait=True, + existing_status=UXAppStatus.RUNNING, + ) + == "" + ) + assert m_subp.call_count == 1 + assert m_sleep.call_count == 0