diff -Nru sosreport-4.2/.cirrus.yml sosreport-4.3/.cirrus.yml --- sosreport-4.2/.cirrus.yml 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/.cirrus.yml 2022-02-15 04:20:20.000000000 +0000 @@ -3,8 +3,8 @@ # Main environment vars to set for all tasks env: - FEDORA_VER: "34" - FEDORA_PRIOR_VER: "33" + FEDORA_VER: "35" + FEDORA_PRIOR_VER: "34" FEDORA_NAME: "fedora-${FEDORA_VER}" FEDORA_PRIOR_NAME: "fedora-${FEDORA_PRIOR_VER}" @@ -20,8 +20,8 @@ # These are generated images pushed to GCP from Red Hat FEDORA_IMAGE_NAME: "f${FEDORA_VER}-server-sos-testing" FEDORA_PRIOR_IMAGE_NAME: "f${FEDORA_PRIOR_VER}-server-sos-testing" - FOREMAN_CENTOS_IMAGE_NAME: "foreman-24-centos-8-sos-testing" - FOREMAN_DEBIAN_IMAGE_NAME: "foreman-24-debian-sos-testing" + FOREMAN_CENTOS_IMAGE_NAME: "foreman-25-centos-8-sos-testing" + FOREMAN_DEBIAN_IMAGE_NAME: "foreman-25-debian-10-sos-testing" # Images exist on GCP already CENTOS_IMAGE_NAME: "centos-stream-8-v20210609" @@ -31,7 +31,7 @@ # Default task timeout timeout_in: 30m -# enable auto cancelling concurrent builds on master when multiple PRs are +# enable auto cancelling concurrent builds on main when multiple PRs are # merged at once auto_cancellation: true @@ -45,15 +45,6 @@ image: alpine/flake8:latest flake_script: flake8 sos -# nose tests, again on the community cluster -nosetests_task: - alias: nosetests - name: "Nosetests" - container: - image: python:slim - setup_script: pip install nose - nose_script: nosetests -v --with-cover --cover-package=sos tests/unittests - # Run a check on newer upstream python versions to check for possible # breaks/changes in common modules. This is not meant to check any of the actual # collections or archive integrity. @@ -92,8 +83,6 @@ image_project: "${PROJECT}" image_name: "${VM_IMAGE_NAME}" type: e2-medium - # minimum disk size is 20 - disk: 20 matrix: - env: PROJECT: ${CENTOS_PROJECT} @@ -118,14 +107,23 @@ remove_sos_script: &remove_sos | if [ $(command -v apt) ]; then apt -y purge sosreport - apt update + apt update --allow-releaseinfo-change apt -y install python3-pip fi if [ $(command -v dnf) ]; then dnf -y remove sos + dnf -y install python3-pip ethtool fi - setup_script: &setup 'pip3 install avocado-framework' - main_script: PYTHONPATH=tests/ avocado run -t stageone tests/{cleaner,collect,report,vendor}_tests + setup_script: &setup 'pip3 install avocado-framework==94.0' + # run the unittests separately as they require a different PYTHONPATH in + # order for the imports to work properly under avocado + unittest_script: PYTHONPATH=. avocado run tests/unittests/ + main_script: PYTHONPATH=tests/ avocado run --test-runner=runner -t stageone tests/{cleaner,collect,report,vendor}_tests + on_failure: + fail_script: &faillogs | + ls -d /var/tmp/avocado* /root/avocado* 2> /dev/null | xargs tar cf sos-fail-logs.tar + log_artifacts: + path: "sos-fail-logs.tar" # IFF the stage one tests all pass, then run stage two for latest distros report_stagetwo_task: @@ -155,7 +153,11 @@ dnf -y install python3-pexpect fi setup_script: *setup - main_script: PYTHONPATH=tests/ avocado run -t stagetwo tests/{cleaner,collect,report,vendor}_tests + main_script: PYTHONPATH=tests/ avocado run --test-runner=runner -t stagetwo tests/{cleaner,collect,report,vendor}_tests + on_failure: + fail_script: *faillogs + log_artifacts: + path: "sos-fail-logs.tar" report_foreman_task: skip: "!changesInclude('.cirrus.yml', '**/{__init__,apache,foreman,foreman_tests,candlepin,pulp,pulpcore}.py')" @@ -169,11 +171,15 @@ - env: PROJECT: ${SOS_PROJECT} VM_IMAGE_NAME: ${FOREMAN_CENTOS_IMAGE_NAME} - FOREMAN_VER: "2.4 - CentOS Stream 8" + FOREMAN_VER: "2.5 - CentOS Stream 8" - env: PROJECT: ${SOS_PROJECT} VM_IMAGE_NAME: ${FOREMAN_DEBIAN_IMAGE_NAME} - FOREMAN_VER: "2.4 - Debian 10" + FOREMAN_VER: "2.5 - Debian 10" remove_sos_script: *remove_sos setup_script: *setup - main_script: PYTHONPATH=tests/ avocado run -t foreman tests/product_tests/foreman/ + main_script: PYTHONPATH=tests/ avocado run --test-runner=runner -t foreman tests/product_tests/foreman/ + on_failure: + fail_script: *faillogs + log_artifacts: + path: "sos-fail-logs.tar" diff -Nru sosreport-4.2/debian/changelog sosreport-4.3/debian/changelog --- sosreport-4.2/debian/changelog 2021-11-01 13:02:20.000000000 +0000 +++ sosreport-4.3/debian/changelog 2022-02-17 12:26:59.000000000 +0000 @@ -1,3 +1,29 @@ +sosreport (4.3-1ubuntu0.20.04.1) focal; urgency=medium + + * New 4.3 upstream. (LP: #1960996) + + * For more details, full release note is available here: + - https://github.com/sosreport/sos/releases/tag/4.3 + + * New patches: + - d/p/0002-fix-setup-py.patch: + Add python sos.help module, it was miss in + upstream release. + - d/p/0003-mention-sos-help-in-sos-manpage.patch: + Fix sos-help manpage. + + * Former patches, now fixed: + - d/p/0002-report-implement_estimate-only.patch + - d/p/0003-ceph-add-support-for-containerized-ceph-setup.patch + - d/p/0004-ceph-split-plugin-by-components.patch + - d/p/0005-openvswitch-get-userspace-datapath-implementations.patch + - d/p/0006-report-check-for-symlink-before-rmtree.patch + + * Remaining patches: + - d/p/0001-debian-change-tmp-dir-location.patch + + -- Nikhil Kshirsagar Thu, 17 Feb 2022 12:26:59 +0000 + sosreport (4.2-1ubuntu0.20.04.1) focal; urgency=medium * New 4.2 upstream release. (LP: #1941745) diff -Nru sosreport-4.2/debian/patches/0002-fix-setup-py.patch sosreport-4.3/debian/patches/0002-fix-setup-py.patch --- sosreport-4.2/debian/patches/0002-fix-setup-py.patch 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/debian/patches/0002-fix-setup-py.patch 2022-02-17 12:17:31.000000000 +0000 @@ -0,0 +1,13 @@ +Index: sos-4.3/setup.py +=================================================================== +--- sos-4.3.orig/setup.py ++++ sos-4.3/setup.py +@@ -97,7 +97,7 @@ setup( + ('config', ['sos.conf']) + ], + packages=[ +- 'sos', 'sos.presets', 'sos.presets.redhat', 'sos.policies', ++ 'sos', 'sos.help', 'sos.presets', 'sos.presets.redhat', 'sos.policies', + 'sos.policies.distros', 'sos.policies.runtimes', + 'sos.policies.package_managers', 'sos.policies.init_systems', + 'sos.report', 'sos.report.plugins', 'sos.collector', diff -Nru sosreport-4.2/debian/patches/0002-report-implement_estimate-only.patch sosreport-4.3/debian/patches/0002-report-implement_estimate-only.patch --- sosreport-4.2/debian/patches/0002-report-implement_estimate-only.patch 2021-11-01 13:02:20.000000000 +0000 +++ sosreport-4.3/debian/patches/0002-report-implement_estimate-only.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,172 +0,0 @@ -From 2f21baf0816464d4725313608dff710ffce84138 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 8 Sep 2021 17:04:58 +0200 -Subject: [PATCH] [report] Implement --estimate-only - -Add report option --estimate-only to estimate disk space requirements -when running a sos report. - -Resolves: #2673 - -Signed-off-by: Pavel Moravec ---- - man/en/sos-report.1 | 13 +++++++- - sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++-- - 2 files changed, 84 insertions(+), 3 deletions(-) - ---- a/man/en/sos-report.1 -+++ b/man/en/sos-report.1 -@@ -14,7 +14,7 @@ - [--preset preset] [--add-preset add_preset]\fR - [--del-preset del_preset] [--desc description]\fR - [--batch] [--build] [--debug] [--dry-run]\fR -- [--label label] [--case-id id]\fR -+ [--estimate-only] [--label label] [--case-id id]\fR - [--threads threads]\fR - [--plugin-timeout TIMEOUT]\fR - [--cmd-timeout TIMEOUT]\fR -@@ -317,6 +317,17 @@ - to understand the actions that sos would have taken without the dry run - option. - .TP -+.B \--estimate-only -+Estimate disk space requirements when running sos report. This can be valuable -+to prevent sosreport working dir to consume all free disk space. No plugin data -+is available at the end. -+ -+Plugins will be collected sequentially, size of collected files and commands outputs -+will be calculated and the plugin files will be immediatelly deleted prior execution -+of the next plugin. This still can consume whole free disk space, though. Please note, -+size estimations may not be accurate for highly utilized systems due to changes between -+an estimate and a real execution. -+.TP - .B \--upload - If specified, attempt to upload the resulting archive to a vendor defined location. - ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -86,6 +86,7 @@ - 'desc': '', - 'domains': [], - 'dry_run': False, -+ 'estimate_only': False, - 'experimental': False, - 'enable_plugins': [], - 'keywords': [], -@@ -137,6 +138,7 @@ - self._args = args - self.sysroot = "/" - self.preset = None -+ self.estimated_plugsizes = {} - - self.print_header() - self._set_debug() -@@ -223,6 +225,11 @@ - help="Description for a new preset",) - report_grp.add_argument("--dry-run", action="store_true", - help="Run plugins but do not collect data") -+ report_grp.add_argument("--estimate-only", action="store_true", -+ help="Approximate disk space requirements for " -+ "a real sos run; disables --clean and " -+ "--collect, sets --threads=1 and " -+ "--no-postproc") - report_grp.add_argument("--experimental", action="store_true", - dest="experimental", default=False, - help="enable experimental plugins") -@@ -693,6 +700,33 @@ - self.all_options.append((plugin, plugin_name, optname, - optparm)) - -+ def _set_estimate_only(self): -+ # set estimate-only mode by enforcing some options settings -+ # and return a corresponding log messages string -+ msg = "\nEstimate-only mode enabled" -+ ext_msg = [] -+ if self.opts.threads > 1: -+ ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ] -+ self.opts.threads = 1 -+ if not self.opts.build: -+ ext_msg += ["--build enabled", ] -+ self.opts.build = True -+ if not self.opts.no_postproc: -+ ext_msg += ["--no-postproc enabled", ] -+ self.opts.no_postproc = True -+ if self.opts.clean: -+ ext_msg += ["--clean disabled", ] -+ self.opts.clean = False -+ if self.opts.upload: -+ ext_msg += ["--upload* options disabled", ] -+ self.opts.upload = False -+ if ext_msg: -+ msg += ", which overrides some options:\n " + "\n ".join(ext_msg) -+ else: -+ msg += "." -+ msg += "\n\n" -+ return msg -+ - def _report_profiles_and_plugins(self): - self.ui_log.info("") - if len(self.loaded_plugins): -@@ -864,10 +898,12 @@ - return True - - def batch(self): -+ msg = self.policy.get_msg() -+ if self.opts.estimate_only: -+ msg += self._set_estimate_only() - if self.opts.batch: -- self.ui_log.info(self.policy.get_msg()) -+ self.ui_log.info(msg) - else: -- msg = self.policy.get_msg() - msg += _("Press ENTER to continue, or CTRL-C to quit.\n") - try: - input(msg) -@@ -1014,6 +1050,22 @@ - self.running_plugs.remove(plugin[1]) - self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() - pool._threads.clear() -+ if self.opts.estimate_only: -+ from pathlib import Path -+ tmpdir_path = Path(self.archive.get_tmp_dir()) -+ self.estimated_plugsizes[plugin[1]] = sum( -+ [f.stat().st_size for f in tmpdir_path.glob('**/*') -+ if (os.path.isfile(f) and not os.path.islink(f))]) -+ # remove whole tmp_dir content - including "sos_commands" and -+ # similar dirs that will be re-created on demand by next plugin -+ # if needed; it is less error-prone approach than skipping -+ # deletion of some dirs but deleting their content -+ for f in os.listdir(self.archive.get_tmp_dir()): -+ f = os.path.join(self.archive.get_tmp_dir(), f) -+ if os.path.isdir(f): -+ rmtree(f) -+ else: -+ os.unlink(f) - return True - - def collect_plugin(self, plugin): -@@ -1333,6 +1385,24 @@ - self.policy.display_results(archive, directory, checksum, - map_file=map_file) - -+ if self.opts.estimate_only: -+ from sos.utilities import get_human_readable -+ _sum = get_human_readable(sum(self.estimated_plugsizes.values())) -+ self.ui_log.info("Estimated disk space requirement for whole " -+ "uncompressed sos report directory: %s" % _sum) -+ bigplugins = sorted(self.estimated_plugsizes.items(), -+ key=lambda x: x[1], reverse=True)[:3] -+ bp_out = ", ".join("%s: %s" % -+ (p, get_human_readable(v, precision=0)) -+ for p, v in bigplugins) -+ self.ui_log.info("Three biggest plugins: %s" % bp_out) -+ self.ui_log.info("") -+ self.ui_log.info("Please note the estimation is relevant to the " -+ "current options.") -+ self.ui_log.info("Be aware that the real disk space requirements " -+ "might be different.") -+ self.ui_log.info("") -+ - if self.opts.upload or self.opts.upload_url: - if not self.opts.build: - try: diff -Nru sosreport-4.2/debian/patches/0003-ceph-add-support-for-containerized-ceph-setup.patch sosreport-4.3/debian/patches/0003-ceph-add-support-for-containerized-ceph-setup.patch --- sosreport-4.2/debian/patches/0003-ceph-add-support-for-containerized-ceph-setup.patch 2021-11-01 13:02:20.000000000 +0000 +++ sosreport-4.3/debian/patches/0003-ceph-add-support-for-containerized-ceph-setup.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,89 +0,0 @@ -From 93da93267a582d8e93e1da55f7e6a6763a6791b8 Mon Sep 17 00:00:00 2001 -From: Teoman ONAY -Date: Fri, 13 Aug 2021 11:18:16 +0200 -Subject: [PATCH] [Ceph] Add support for containerized Ceph setup - -RH Ceph 4 can either be installed as RPM or as containers. The changes -permits to collect the ceph config on both kind of setup. - -Signed-off-by: Teoman ONAY ---- - sos/report/plugins/ceph.py | 41 ++++++++++++++++++++++++++++++++++---- - 1 file changed, 37 insertions(+), 4 deletions(-) - ---- a/sos/report/plugins/ceph.py -+++ b/sos/report/plugins/ceph.py -@@ -8,6 +8,7 @@ - - from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin - from socket import gethostname -+import re - - - class Ceph(Plugin, RedHatPlugin, UbuntuPlugin): -@@ -15,7 +16,8 @@ - short_desc = 'CEPH distributed storage' - - plugin_name = 'ceph' -- profiles = ('storage', 'virt') -+ profiles = ('storage', 'virt', 'container') -+ containers = ('ceph-(mon|rgw|osd)*',) - ceph_hostname = gethostname() - - packages = ( -@@ -37,6 +39,12 @@ - 'ceph-osd@*' - ) - -+ # This check will enable the plugin regardless of being -+ # containerized or not -+ files = ( -+ '/etc/ceph/ceph.conf', -+ ) -+ - def setup(self): - all_logs = self.get_option("all_logs") - -@@ -117,9 +125,9 @@ - "time-sync-status", - ] - -- self.add_cmd_output([ -- "ceph %s" % s for s in ceph_cmds -- ]) -+ ceph_osd_cmds = [ -+ "ceph-volume lvm list", -+ ] - - self.add_cmd_output([ - "ceph %s --format json-pretty" % s for s in ceph_cmds -@@ -138,4 +146,29 @@ - "/etc/ceph/*bindpass*" - ]) - -+ # If containerized, run commands in containers -+ containers_list = self.get_all_containers_by_regex("ceph-*") -+ if containers_list: -+ # Avoid retrieving multiple times the same data -+ got_ceph_cmds = False -+ for container in containers_list: -+ if re.match("ceph-(mon|rgw|osd)", container[1]) and \ -+ not got_ceph_cmds: -+ self.add_cmd_output([ -+ self.fmt_container_cmd(container[1], "ceph %s" % s) -+ for s in ceph_cmds -+ ]) -+ got_ceph_cmds = True -+ if re.match("ceph-osd", container[1]): -+ self.add_cmd_output([ -+ self.fmt_container_cmd(container[1], "%s" % s) -+ for s in ceph_osd_cmds -+ ]) -+ break -+ # Not containerized -+ else: -+ self.add_cmd_output([ -+ "ceph %s" % s for s in ceph_cmds -+ ]) -+ - # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/debian/patches/0003-mention-sos-help-in-sos-manpage.patch sosreport-4.3/debian/patches/0003-mention-sos-help-in-sos-manpage.patch --- sosreport-4.2/debian/patches/0003-mention-sos-help-in-sos-manpage.patch 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/debian/patches/0003-mention-sos-help-in-sos-manpage.patch 2022-02-17 12:25:39.000000000 +0000 @@ -0,0 +1,42 @@ +Index: sos-4.3/man/en/sos.1 +=================================================================== +--- sos-4.3.orig/man/en/sos.1 ++++ sos-4.3/man/en/sos.1 +@@ -67,6 +67,14 @@ May be invoked via either \fBsos clean\f + or via the \fB--clean\fR, \fB--cleaner\fR or \fB --mask\fR options + for \fBreport\fR and \fBcollect\fR. + ++.TP ++.B help ++This subcommand is used to retrieve more detailed information on the various SoS ++commands and components than is directly available in either other manpages or ++--help output. ++ ++See \fB sos help --help\fR and \fB man sos-help\fR for more information. ++ + .SH GLOBAL OPTIONS + sos components provide their own set of options, however the following are available + to be set across all components. +Index: sos-4.3/setup.py +=================================================================== +--- sos-4.3.orig/setup.py ++++ sos-4.3/setup.py +@@ -90,7 +90,7 @@ setup( + ('share/man/man1', ['man/en/sosreport.1', 'man/en/sos-report.1', + 'man/en/sos.1', 'man/en/sos-collect.1', + 'man/en/sos-collector.1', 'man/en/sos-clean.1', +- 'man/en/sos-mask.1']), ++ 'man/en/sos-mask.1', 'man/en/sos-help.1']), + ('share/man/man5', ['man/en/sos.conf.5']), + ('share/licenses/sos', ['LICENSE']), + ('share/doc/sos', ['AUTHORS', 'README.md']), +@@ -102,7 +102,8 @@ setup( + 'sos.policies.package_managers', 'sos.policies.init_systems', + 'sos.report', 'sos.report.plugins', 'sos.collector', + 'sos.collector.clusters', 'sos.collector.transports', 'sos.cleaner', +- 'sos.cleaner.mappings', 'sos.cleaner.parsers', 'sos.cleaner.archives' ++ 'sos.cleaner.mappings', 'sos.cleaner.parsers', 'sos.cleaner.archives', ++ 'sos.help' + ], + cmdclass=cmdclass, + command_options=command_options, diff -Nru sosreport-4.2/debian/patches/0004-ceph-split-plugin-by-components.patch sosreport-4.3/debian/patches/0004-ceph-split-plugin-by-components.patch --- sosreport-4.2/debian/patches/0004-ceph-split-plugin-by-components.patch 2021-11-01 13:02:20.000000000 +0000 +++ sosreport-4.3/debian/patches/0004-ceph-split-plugin-by-components.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,652 +0,0 @@ -From d8a071bf85fecf41a86efea5ff574b7117bff151 Mon Sep 17 00:00:00 2001 -From: Nikhil Kshirsagar -Date: Fri, 17 Sep 2021 21:29:34 +0530 -Subject: [PATCH] [ceph] split the ceph plugin - -This work distributes the ceph plugin into plugins -for individual ceph components (mon,osd,mds,rgw,mgr), -so that additional data collection can then be added -in each of the components. Work for additional data -collection in each component plugin will follow in -a later commit. - -Closes: #1945 - -Signed-off-by: Nikhil Kshirsagar ---- - sos/report/plugins/ceph_common.py | 85 +++++++++++++ - sos/report/plugins/ceph_mds.py | 48 ++++++++ - sos/report/plugins/ceph_mgr.py | 81 +++++++++++++ - sos/report/plugins/{ceph.py => ceph_mon.py} | 128 ++++++-------------- - sos/report/plugins/ceph_osd.py | 58 +++++++++ - sos/report/plugins/ceph_rgw.py | 41 +++++++ - 6 files changed, 347 insertions(+), 94 deletions(-) - create mode 100644 sos/report/plugins/ceph_common.py - create mode 100644 sos/report/plugins/ceph_mds.py - create mode 100644 sos/report/plugins/ceph_mgr.py - rename sos/report/plugins/{ceph.py => ceph_mon.py} (52%) - create mode 100644 sos/report/plugins/ceph_osd.py - create mode 100644 sos/report/plugins/ceph_rgw.py - ---- /dev/null -+++ b/sos/report/plugins/ceph_common.py -@@ -0,0 +1,85 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -+from socket import gethostname -+ -+ -+class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin): -+ -+ short_desc = 'CEPH common' -+ -+ plugin_name = 'ceph_common' -+ profiles = ('storage', 'virt', 'container') -+ -+ containers = ('ceph-(mon|rgw|osd).*',) -+ ceph_hostname = gethostname() -+ -+ packages = ( -+ 'ceph', -+ 'ceph-mds', -+ 'ceph-common', -+ 'libcephfs1', -+ 'ceph-fs-common', -+ 'calamari-server', -+ 'librados2' -+ ) -+ -+ services = ( -+ 'ceph-nfs@pacemaker', -+ 'ceph-mds@%s' % ceph_hostname, -+ 'ceph-mon@%s' % ceph_hostname, -+ 'ceph-mgr@%s' % ceph_hostname, -+ 'ceph-radosgw@*', -+ 'ceph-osd@*' -+ ) -+ -+ # This check will enable the plugin regardless of being -+ # containerized or not -+ files = ('/etc/ceph/ceph.conf',) -+ -+ def setup(self): -+ all_logs = self.get_option("all_logs") -+ -+ self.add_file_tags({ -+ '.*/ceph.conf': 'ceph_conf', -+ '/var/log/ceph/ceph.log.*': 'ceph_log', -+ }) -+ -+ if not all_logs: -+ self.add_copy_spec("/var/log/calamari/*.log",) -+ else: -+ self.add_copy_spec("/var/log/calamari",) -+ -+ self.add_copy_spec([ -+ "/var/log/ceph/ceph.log", -+ "/var/log/ceph/ceph.audit.log*", -+ "/var/log/calamari/*.log", -+ "/etc/ceph/", -+ "/etc/calamari/", -+ "/var/lib/ceph/tmp/", -+ ]) -+ -+ self.add_cmd_output([ -+ "ceph -v", -+ ]) -+ -+ self.add_forbidden_path([ -+ "/etc/ceph/*keyring*", -+ "/var/lib/ceph/*keyring*", -+ "/var/lib/ceph/*/*keyring*", -+ "/var/lib/ceph/*/*/*keyring*", -+ "/var/lib/ceph/osd", -+ "/var/lib/ceph/mon", -+ # Excludes temporary ceph-osd mount location like -+ # /var/lib/ceph/tmp/mnt.XXXX from sos collection. -+ "/var/lib/ceph/tmp/*mnt*", -+ "/etc/ceph/*bindpass*" -+ ]) -+ -+# vim: set et ts=4 sw=4 : ---- /dev/null -+++ b/sos/report/plugins/ceph_mds.py -@@ -0,0 +1,48 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -+import glob -+ -+ -+class CephMDS(Plugin, RedHatPlugin, UbuntuPlugin): -+ short_desc = 'CEPH mds' -+ plugin_name = 'ceph_mds' -+ profiles = ('storage', 'virt', 'container') -+ containers = ('ceph-fs.*',) -+ -+ def check_enabled(self): -+ return True if glob.glob('/var/lib/ceph/mds/*/*') else False -+ -+ def setup(self): -+ self.add_file_tags({ -+ '/var/log/ceph/ceph-mds.*.log': 'ceph_mds_log', -+ }) -+ -+ self.add_copy_spec([ -+ "/var/log/ceph.log", -+ "/var/log/ceph/ceph-mds*.log", -+ "/var/lib/ceph/bootstrap-mds/", -+ "/var/lib/ceph/mds/", -+ "/run/ceph/ceph-mds*", -+ ]) -+ -+ self.add_forbidden_path([ -+ "/etc/ceph/*keyring*", -+ "/var/lib/ceph/*keyring*", -+ "/var/lib/ceph/*/*keyring*", -+ "/var/lib/ceph/*/*/*keyring*", -+ "/var/lib/ceph/osd", -+ "/var/lib/ceph/mon", -+ # Excludes temporary ceph-osd mount location like -+ # /var/lib/ceph/tmp/mnt.XXXX from sos collection. -+ "/var/lib/ceph/tmp/*mnt*", -+ "/etc/ceph/*bindpass*" -+ ]) -+ -+# vim: set et ts=4 sw=4 : ---- /dev/null -+++ b/sos/report/plugins/ceph_mgr.py -@@ -0,0 +1,81 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -+import glob -+ -+ -+class CephMGR(Plugin, RedHatPlugin, UbuntuPlugin): -+ -+ short_desc = 'CEPH mgr' -+ -+ plugin_name = 'ceph_mgr' -+ profiles = ('storage', 'virt', 'container') -+ -+ containers = ('ceph-mgr.*',) -+ -+ def check_enabled(self): -+ return True if glob.glob('/var/lib/ceph/mgr/*/*') else False -+ -+ def setup(self): -+ self.add_file_tags({ -+ '/var/log/ceph/ceph-mgr.*.log': 'ceph_mgr_log', -+ }) -+ -+ self.add_copy_spec([ -+ "/var/log/ceph/ceph-mgr*.log", -+ "/var/lib/ceph/mgr/", -+ "/var/lib/ceph/bootstrap-mgr/", -+ "/run/ceph/ceph-mgr*", -+ ]) -+ -+ # more commands to be added later -+ self.add_cmd_output([ -+ "ceph balancer status", -+ "ceph mgr metadata", -+ ]) -+ -+ # more commands to be added later -+ ceph_cmds = [ -+ "mgr module ls", -+ "mgr dump", -+ ] -+ -+ self.add_cmd_output([ -+ "ceph %s --format json-pretty" % s for s in ceph_cmds -+ ], subdir="json_output", tags="insights_ceph_health_detail") -+ -+ self.add_forbidden_path([ -+ "/etc/ceph/*keyring*", -+ "/var/lib/ceph/*keyring*", -+ "/var/lib/ceph/*/*keyring*", -+ "/var/lib/ceph/*/*/*keyring*", -+ "/var/lib/ceph/osd", -+ "/var/lib/ceph/mon", -+ # Excludes temporary ceph-osd mount location like -+ # /var/lib/ceph/tmp/mnt.XXXX from sos collection. -+ "/var/lib/ceph/tmp/*mnt*", -+ "/etc/ceph/*bindpass*", -+ ]) -+ -+ # If containerized, run commands in containers -+ containers_list = self.get_all_containers_by_regex("ceph-mgr*") -+ if containers_list: -+ for container in containers_list: -+ self.add_cmd_output([ -+ self.fmt_container_cmd(container[1], "ceph %s" % s) -+ for s in ceph_cmds -+ ]) -+ break -+ # Not containerized -+ else: -+ self.add_cmd_output([ -+ "ceph %s" % s for s in ceph_cmds -+ ]) -+ -+# vim: set et ts=4 sw=4 : ---- a/sos/report/plugins/ceph.py -+++ /dev/null -@@ -1,174 +0,0 @@ --# This file is part of the sos project: https://github.com/sosreport/sos --# --# This copyrighted material is made available to anyone wishing to use, --# modify, copy, or redistribute it subject to the terms and conditions of --# version 2 of the GNU General Public License. --# --# See the LICENSE file in the source distribution for further information. -- --from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin --from socket import gethostname --import re -- -- --class Ceph(Plugin, RedHatPlugin, UbuntuPlugin): -- -- short_desc = 'CEPH distributed storage' -- -- plugin_name = 'ceph' -- profiles = ('storage', 'virt', 'container') -- containers = ('ceph-(mon|rgw|osd)*',) -- ceph_hostname = gethostname() -- -- packages = ( -- 'ceph', -- 'ceph-mds', -- 'ceph-common', -- 'libcephfs1', -- 'ceph-fs-common', -- 'calamari-server', -- 'librados2' -- ) -- -- services = ( -- 'ceph-nfs@pacemaker', -- 'ceph-mds@%s' % ceph_hostname, -- 'ceph-mon@%s' % ceph_hostname, -- 'ceph-mgr@%s' % ceph_hostname, -- 'ceph-radosgw@*', -- 'ceph-osd@*' -- ) -- -- # This check will enable the plugin regardless of being -- # containerized or not -- files = ( -- '/etc/ceph/ceph.conf', -- ) -- -- def setup(self): -- all_logs = self.get_option("all_logs") -- -- self.add_file_tags({ -- '.*/ceph.conf': 'ceph_conf', -- '/var/log/ceph/ceph.log.*': 'ceph_log', -- '/var/log/ceph/ceph-osd.*.log': 'ceph_osd_log' -- }) -- -- if not all_logs: -- self.add_copy_spec([ -- "/var/log/ceph/*.log", -- "/var/log/radosgw/*.log", -- "/var/log/calamari/*.log" -- ]) -- else: -- self.add_copy_spec([ -- "/var/log/ceph/", -- "/var/log/calamari", -- "/var/log/radosgw" -- ]) -- -- self.add_copy_spec([ -- "/etc/ceph/", -- "/etc/calamari/", -- "/var/lib/ceph/", -- "/run/ceph/" -- ]) -- -- self.add_cmd_output([ -- "ceph mon stat", -- "ceph mon_status", -- "ceph quorum_status", -- "ceph mgr module ls", -- "ceph mgr metadata", -- "ceph balancer status", -- "ceph osd metadata", -- "ceph osd erasure-code-profile ls", -- "ceph report", -- "ceph osd crush show-tunables", -- "ceph-disk list", -- "ceph versions", -- "ceph features", -- "ceph insights", -- "ceph osd crush dump", -- "ceph -v", -- "ceph-volume lvm list", -- "ceph crash stat", -- "ceph crash ls", -- "ceph config log", -- "ceph config generate-minimal-conf", -- "ceph config-key dump", -- ]) -- -- ceph_cmds = [ -- "status", -- "health detail", -- "osd tree", -- "osd stat", -- "osd df tree", -- "osd dump", -- "osd df", -- "osd perf", -- "osd blocked-by", -- "osd pool ls detail", -- "osd pool autoscale-status", -- "osd numa-status", -- "device ls", -- "mon dump", -- "mgr dump", -- "mds stat", -- "df", -- "df detail", -- "fs ls", -- "fs dump", -- "pg dump", -- "pg stat", -- "time-sync-status", -- ] -- -- ceph_osd_cmds = [ -- "ceph-volume lvm list", -- ] -- -- self.add_cmd_output([ -- "ceph %s --format json-pretty" % s for s in ceph_cmds -- ], subdir="json_output", tags="insights_ceph_health_detail") -- -- self.add_forbidden_path([ -- "/etc/ceph/*keyring*", -- "/var/lib/ceph/*keyring*", -- "/var/lib/ceph/*/*keyring*", -- "/var/lib/ceph/*/*/*keyring*", -- "/var/lib/ceph/osd", -- "/var/lib/ceph/mon", -- # Excludes temporary ceph-osd mount location like -- # /var/lib/ceph/tmp/mnt.XXXX from sos collection. -- "/var/lib/ceph/tmp/*mnt*", -- "/etc/ceph/*bindpass*" -- ]) -- -- # If containerized, run commands in containers -- containers_list = self.get_all_containers_by_regex("ceph-*") -- if containers_list: -- # Avoid retrieving multiple times the same data -- got_ceph_cmds = False -- for container in containers_list: -- if re.match("ceph-(mon|rgw|osd)", container[1]) and \ -- not got_ceph_cmds: -- self.add_cmd_output([ -- self.fmt_container_cmd(container[1], "ceph %s" % s) -- for s in ceph_cmds -- ]) -- got_ceph_cmds = True -- if re.match("ceph-osd", container[1]): -- self.add_cmd_output([ -- self.fmt_container_cmd(container[1], "%s" % s) -- for s in ceph_osd_cmds -- ]) -- break -- # Not containerized -- else: -- self.add_cmd_output([ -- "ceph %s" % s for s in ceph_cmds -- ]) -- --# vim: set et ts=4 sw=4 : ---- /dev/null -+++ b/sos/report/plugins/ceph_mon.py -@@ -0,0 +1,114 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -+import glob -+ -+ -+class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): -+ -+ short_desc = 'CEPH mon' -+ -+ plugin_name = 'ceph_mon' -+ profiles = ('storage', 'virt', 'container') -+ containers = ('ceph-mon.*',) -+ -+ def check_enabled(self): -+ return True if glob.glob('/var/lib/ceph/mon/*/*') else False -+ -+ def setup(self): -+ self.add_file_tags({ -+ '.*/ceph.conf': 'ceph_conf', -+ '/var/log/ceph/ceph-mon.*.log': 'ceph_mon_log' -+ }) -+ -+ self.add_copy_spec([ -+ "/var/log/ceph/ceph-mon*.log", -+ "/var/lib/ceph/mon/", -+ "/run/ceph/ceph-mon*" -+ ]) -+ -+ self.add_cmd_output([ -+ "ceph mon stat", -+ "ceph quorum_status", -+ "ceph report", -+ "ceph-disk list", -+ "ceph versions", -+ "ceph features", -+ "ceph insights", -+ "ceph crash stat", -+ "ceph crash ls", -+ "ceph config log", -+ "ceph config generate-minimal-conf", -+ "ceph config-key dump", -+ "ceph mon_status", -+ "ceph osd metadata", -+ "ceph osd erasure-code-profile ls", -+ "ceph osd crush show-tunables", -+ "ceph osd crush dump" -+ ]) -+ -+ ceph_cmds = [ -+ "mon dump", -+ "status", -+ "health detail", -+ "device ls", -+ "df", -+ "df detail", -+ "fs ls", -+ "fs dump", -+ "pg dump", -+ "pg stat", -+ "time-sync-status", -+ "osd tree", -+ "osd stat", -+ "osd df tree", -+ "osd dump", -+ "osd df", -+ "osd perf", -+ "osd blocked-by", -+ "osd pool ls detail", -+ "osd pool autoscale-status", -+ "mds stat", -+ "osd numa-status" -+ ] -+ -+ self.add_cmd_output([ -+ "ceph %s --format json-pretty" % s for s in ceph_cmds -+ ], subdir="json_output", tags="insights_ceph_health_detail") -+ -+ # these can be cleaned up too but leaving them for safety for now -+ self.add_forbidden_path([ -+ "/etc/ceph/*keyring*", -+ "/var/lib/ceph/*keyring*", -+ "/var/lib/ceph/*/*keyring*", -+ "/var/lib/ceph/*/*/*keyring*", -+ "/var/lib/ceph/osd", -+ "/var/lib/ceph/mon", -+ # Excludes temporary ceph-osd mount location like -+ # /var/lib/ceph/tmp/mnt.XXXX from sos collection. -+ "/var/lib/ceph/tmp/*mnt*", -+ "/etc/ceph/*bindpass*" -+ ]) -+ -+ # If containerized, run commands in containers -+ containers_list = self.get_all_containers_by_regex("ceph-mon*") -+ if containers_list: -+ for container in containers_list: -+ self.add_cmd_output([ -+ self.fmt_container_cmd(container[1], "ceph %s" % s) -+ for s in ceph_cmds -+ ]) -+ break -+ # Not containerized but still mon node -+ else: -+ self.add_cmd_output([ -+ "ceph %s" % s for s in ceph_cmds -+ ]) -+ -+# vim: set et ts=4 sw=4 : ---- /dev/null -+++ b/sos/report/plugins/ceph_osd.py -@@ -0,0 +1,58 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -+import glob -+ -+ -+class CephOSD(Plugin, RedHatPlugin, UbuntuPlugin): -+ -+ short_desc = 'CEPH osd' -+ -+ plugin_name = 'ceph_osd' -+ profiles = ('storage', 'virt', 'container') -+ containers = ('ceph-osd.*',) -+ -+ def check_enabled(self): -+ return True if glob.glob('/var/lib/ceph/osd/*/*') else False -+ -+ def setup(self): -+ self.add_file_tags({ -+ '/var/log/ceph/ceph-osd.*.log': 'ceph_osd_log', -+ }) -+ -+ # Only collect OSD specific files -+ self.add_copy_spec([ -+ "/var/log/ceph/ceph-osd*.log", -+ "/var/log/ceph/ceph-volume*.log", -+ -+ "/var/lib/ceph/osd/", -+ "/var/lib/ceph/bootstrap-osd/", -+ -+ "/run/ceph/ceph-osd*" -+ ]) -+ -+ self.add_cmd_output([ -+ "ceph-disk list", -+ "ceph-volume lvm list", -+ ]) -+ -+ self.add_forbidden_path([ -+ "/etc/ceph/*keyring*", -+ "/var/lib/ceph/*keyring*", -+ "/var/lib/ceph/*/*keyring*", -+ "/var/lib/ceph/*/*/*keyring*", -+ "/var/lib/ceph/osd", -+ "/var/lib/ceph/mon", -+ # Excludes temporary ceph-osd mount location like -+ # /var/lib/ceph/tmp/mnt.XXXX from sos collection. -+ "/var/lib/ceph/tmp/*mnt*", -+ "/etc/ceph/*bindpass*" -+ ]) -+ -+# vim: set et ts=4 sw=4 : ---- /dev/null -+++ b/sos/report/plugins/ceph_rgw.py -@@ -0,0 +1,41 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -+import glob -+ -+ -+class CephRGW(Plugin, RedHatPlugin, UbuntuPlugin): -+ -+ short_desc = 'CEPH rgw' -+ -+ plugin_name = 'ceph_rgw' -+ profiles = ('storage', 'virt', 'container', 'webserver') -+ containers = ('ceph-rgw.*',) -+ -+ def check_enabled(self): -+ return True if glob.glob('/var/lib/ceph/radosgw/*/*') else False -+ -+ def setup(self): -+ self.add_copy_spec('/var/log/ceph/ceph-client.rgw*.log', -+ tags='ceph_rgw_log') -+ -+ self.add_forbidden_path([ -+ "/etc/ceph/*keyring*", -+ "/var/lib/ceph/*keyring*", -+ "/var/lib/ceph/*/*keyring*", -+ "/var/lib/ceph/*/*/*keyring*", -+ "/var/lib/ceph/osd", -+ "/var/lib/ceph/mon", -+ # Excludes temporary ceph-osd mount location like -+ # /var/lib/ceph/tmp/mnt.XXXX from sos collection. -+ "/var/lib/ceph/tmp/*mnt*", -+ "/etc/ceph/*bindpass*" -+ ]) -+ -+# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/debian/patches/0005-openvswitch-get-userspace-datapath-implementations.patch sosreport-4.3/debian/patches/0005-openvswitch-get-userspace-datapath-implementations.patch --- sosreport-4.2/debian/patches/0005-openvswitch-get-userspace-datapath-implementations.patch 2021-11-01 13:02:20.000000000 +0000 +++ sosreport-4.3/debian/patches/0005-openvswitch-get-userspace-datapath-implementations.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,32 +0,0 @@ -From bc77edfa5fec83b94026b735b4a581d1067bac21 Mon Sep 17 00:00:00 2001 -From: Kevin Traynor -Date: Thu, 26 Aug 2021 12:49:34 +0100 -Subject: [PATCH] [openvswitch] Get userspace datapath implementations - -OVS has options for the userspace datapath so certain functionality -can use implementations based on SIMD instructions if available. - -Add some commands which show the implementations available and used. - -Signed-off-by: Kevin Traynor ---- - sos/report/plugins/openvswitch.py | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - ---- a/sos/report/plugins/openvswitch.py -+++ b/sos/report/plugins/openvswitch.py -@@ -131,7 +131,13 @@ - # Capture OVS offload enabled flows - "ovs-dpctl dump-flows --name -m type=offloaded", - # Capture OVS slowdatapth flows -- "ovs-dpctl dump-flows --name -m type=ovs" -+ "ovs-dpctl dump-flows --name -m type=ovs", -+ # Capture dpcls implementations -+ "ovs-appctl dpif-netdev/subtable-lookup-prio-get", -+ # Capture dpif implementations -+ "ovs-appctl dpif-netdev/dpif-impl-get", -+ # Capture miniflow extract implementations -+ "ovs-appctl dpif-netdev/miniflow-parser-get" - ]) - - # Gather systemd services logs diff -Nru sosreport-4.2/debian/patches/0006-report-check-for-symlink-before-rmtree.patch sosreport-4.3/debian/patches/0006-report-check-for-symlink-before-rmtree.patch --- sosreport-4.2/debian/patches/0006-report-check-for-symlink-before-rmtree.patch 2021-11-01 13:02:20.000000000 +0000 +++ sosreport-4.3/debian/patches/0006-report-check-for-symlink-before-rmtree.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -From 921218bdefad1ad95bea2dbc639459f26a5f6113 Mon Sep 17 00:00:00 2001 -From: Eric Desrochers -Date: Tue, 19 Oct 2021 12:18:40 -0400 -Subject: [PATCH] [report] check for symlink before rmtree when opt - estimate-only is use - -Check if the dir is also symlink before performing rmtree() -method so that unlink() method can be used instead. - -Traceback (most recent call last): - File "./bin/sos", line 22, in - sos.execute() - File "/tmp/sos/sos/__init__.py", line 186, in execute - self._component.execute() -OSError: Cannot call rmtree on a symbolic link - -Closes: #2727 - -Signed-off-by: Eric Desrochers ---- - sos/report/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -1062,7 +1062,7 @@ - # deletion of some dirs but deleting their content - for f in os.listdir(self.archive.get_tmp_dir()): - f = os.path.join(self.archive.get_tmp_dir(), f) -- if os.path.isdir(f): -+ if os.path.isdir(f) and not os.path.islink(f): - rmtree(f) - else: - os.unlink(f) diff -Nru sosreport-4.2/debian/patches/series sosreport-4.3/debian/patches/series --- sosreport-4.2/debian/patches/series 2021-11-01 13:02:20.000000000 +0000 +++ sosreport-4.3/debian/patches/series 2022-02-17 12:19:40.000000000 +0000 @@ -1,6 +1,3 @@ 0001-debian-change-tmp-dir-location.patch -0002-report-implement_estimate-only.patch -0003-ceph-add-support-for-containerized-ceph-setup.patch -0004-ceph-split-plugin-by-components.patch -0005-openvswitch-get-userspace-datapath-implementations.patch -0006-report-check-for-symlink-before-rmtree.patch +0002-fix-setup-py.patch +0003-mention-sos-help-in-sos-manpage.patch diff -Nru sosreport-4.2/docs/conf.py sosreport-4.3/docs/conf.py --- sosreport-4.2/docs/conf.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/docs/conf.py 2022-02-15 04:20:20.000000000 +0000 @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = '4.2' +version = '4.3' # The full version, including alpha/beta/rc tags. -release = '4.2' +release = '4.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff -Nru sosreport-4.2/docs/index.rst sosreport-4.3/docs/index.rst --- sosreport-4.2/docs/index.rst 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/docs/index.rst 2022-02-15 04:20:20.000000000 +0000 @@ -13,7 +13,7 @@ For the latest version, to contribute, and for more information, please visit the project pages or join the mailing list. -To clone the current master (development) branch run: +To clone the current main (development) branch run: .. code:: @@ -32,7 +32,7 @@ Patches and pull requests ^^^^^^^^^^^^^^^^^^^^^^^^^ -Patches can be submitted via the mailing list or as GitHub pull requests. If using GitHub please make sure your branch applies to the current master as a 'fast forward' merge (i.e. without creating a merge commit). Use the git rebase command to update your branch to the current master if necessary. +Patches can be submitted via the mailing list or as GitHub pull requests. If using GitHub please make sure your branch applies to the current main branch as a 'fast forward' merge (i.e. without creating a merge commit). Use the git rebase command to update your branch to the current main branch if necessary. Documentation ============= diff -Nru sosreport-4.2/.editorconfig sosreport-4.3/.editorconfig --- sosreport-4.2/.editorconfig 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/.editorconfig 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,9 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.py] +indent_style = space +indent_size = 4 diff -Nru sosreport-4.2/man/en/sos.1 sosreport-4.3/man/en/sos.1 --- sosreport-4.2/man/en/sos.1 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sos.1 2022-02-15 04:20:20.000000000 +0000 @@ -37,7 +37,7 @@ .B collect Collect is used to capture reports on multiple systems simultaneously. These systems can either be defined by the user at the command line and/or defined by -clustering software that exists either on the local system or on a "master" system +clustering software that exists either on the local system or on a "primary" system that is able to inform about other nodes in the cluster. When running collect, sos report will be run on the remote nodes, and then the diff -Nru sosreport-4.2/man/en/sos-clean.1 sosreport-4.3/man/en/sos-clean.1 --- sosreport-4.2/man/en/sos-clean.1 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sos-clean.1 2022-02-15 04:20:20.000000000 +0000 @@ -10,6 +10,7 @@ [\-\-jobs] [\-\-no-update] [\-\-keep-binary-files] + [\-\-archive-type] .SH DESCRIPTION \fBsos clean\fR or \fBsos mask\fR is an sos subcommand used to obfuscate sensitive information from @@ -88,6 +89,31 @@ a third party. Default: False (remove encountered binary files) +.TP +.B \-\-archive-type TYPE +Specify the type of archive that TARGET was generated as. +When sos inspects a TARGET archive, it tries to identify what type of archive it is. +For example, it may be a report generated by \fBsos report\fR, or a collection of those +reports generated by \fBsos collect\fR, which require separate approaches. + +This option may be useful if a given TARGET archive is known to be of a specific type, +but due to unknown reasons or some malformed/missing information in the archive directly, +that is not properly identified by sos. + +The following are accepted values for this option: + + \fBauto\fR Automatically detect the archive type + \fBreport\fR An archive generated by \fBsos report\fR + \fBcollect\fR An archive generated by \fBsos collect\fR + \fBinsights\fR An archive generated by the \fBinsights-client\fR package + +The following may also be used, however note that these do not attempt to pre-load +any information from the archives into the parsers. This means that, among other limitations, +items like host and domain names may not be obfuscated unless an obfuscated mapping already exists +on the system from a previous execution. + + \fBdata-dir\fR A plain directory on the filesystem. + \fBtarball\fR A generic tar archive not associated with any known tool .SH SEE ALSO .BR sos (1) diff -Nru sosreport-4.2/man/en/sos-collect.1 sosreport-4.3/man/en/sos-collect.1 --- sosreport-4.2/man/en/sos-collect.1 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sos-collect.1 2022-02-15 04:20:20.000000000 +0000 @@ -11,6 +11,7 @@ [\-\-chroot CHROOT] [\-\-case\-id CASE_ID] [\-\-cluster\-type CLUSTER_TYPE] + [\-\-container\-runtime RUNTIME] [\-e ENABLE_PLUGINS] [--encrypt-key KEY]\fR [--encrypt-pass PASS]\fR @@ -25,7 +26,7 @@ [\-\-nodes NODES] [\-\-no\-pkg\-check] [\-\-no\-local] - [\-\-master MASTER] + [\-\-primary PRIMARY] [\-\-image IMAGE] [\-\-force-pull-image] [\-\-registry-user USER] @@ -43,6 +44,7 @@ [\-\-sos-cmd SOS_CMD] [\-t|\-\-threads THREADS] [\-\-timeout TIMEOUT] + [\-\-transport TRANSPORT] [\-\-tmp\-dir TMP_DIR] [\-v|\-\-verbose] [\-\-verify] @@ -54,7 +56,7 @@ them in a single useful tar archive. sos collect can be run either on a workstation that has SSH key authentication setup -for the nodes in a given cluster, or from a "master" node in a cluster that has SSH +for the nodes in a given cluster, or from a "primary" node in a cluster that has SSH keys configured for the other nodes. Some sosreport options are supported by sos-collect and are passed directly to @@ -99,7 +101,7 @@ \fB\-\-cluster\-type\fR CLUSTER_TYPE When run by itself, sos collect will attempt to identify the type of cluster at play. This is done by checking package or configuration information against the localhost, or -the master node if \fB"--master"\fR is supplied. +the primary node if \fB"--primary"\fR is supplied. Setting \fB--cluster-type\fR skips this step and forcibly sets a particular profile. @@ -112,6 +114,11 @@ to be run, and thus set sosreport options and attempt to determine a list of nodes using that profile. .TP +\fB\-\-container\-runtime\fR RUNTIME +\fB sos report\fR option. Using this with \fBcollect\fR will pass this option thru +to nodes with sos version 4.3 or later. This option controls the default container +runtime plugins will use for collections. See \fBman sos-report\fR. +.TP \fB\-e\fR ENABLE_PLUGINS, \fB\-\-enable\-plugins\fR ENABLE_PLUGINS Sosreport option. Use this to enable a plugin that would otherwise not be run. @@ -152,10 +159,10 @@ \fB\-\-group\fR GROUP Specify an existing host group definition to use. -Host groups are pre-defined settings for the cluster-type, master, and nodes options +Host groups are pre-defined settings for the cluster-type, primary node, and nodes options saved in JSON-formatted files under /var/lib/sos collect/. -If cluster_type and/or master are set in the group, sos collect behaves as if +If cluster_type and/or primary are set in the group, sos collect behaves as if these values were specified on the command-line. If nodes is defined, sos collect \fBextends\fR the \fB\-\-nodes\fR option, if set, @@ -171,7 +178,7 @@ Save the results of this run of sos collect to a host group definition. sos-colllector will write a JSON-formatted file with name GROUP to /var/lib/sos collect/ -with the settings for cluster-type, master, and the node list as discovered by cluster enumeration. +with the settings for cluster-type, primary, and the node list as discovered by cluster enumeration. Note that this means regexes are not directly saved to host groups, but the results of matching against those regexes are. .TP @@ -234,20 +241,20 @@ Do not perform package checks. Most cluster profiles check against installed packages to determine if the cluster profile should be applied or not. -Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the master/local node. +Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the primary/local node. .TP \fB\-\-no\-local\fR Do not collect a sosreport from the local system. -If \fB--master\fR is not supplied, it is assumed that the host running sosreport is part of +If \fB--primary\fR is not supplied, it is assumed that the host running sosreport is part of the cluster that is to be collected. Use this option to skip collection of a local sosreport. -This option is NOT needed if \fB--master\fR is provided. +This option is NOT needed if \fB--primary\fR is provided. .TP -\fB\-\-master\fR MASTER -Specify a master node for the cluster. +\fB\-\-primary\fR PRIMARY +Specify a primary node IP address or hostname for the cluster. -If provided, then sos collect will check the master node, not localhost, for determining +If provided, then sos collect will check the primary node, not localhost, for determining the type of cluster in use. .TP \fB\-\-image IMAGE\fR @@ -351,6 +358,21 @@ Default is 180 seconds. .TP +\fB\-\-transport\fR TRANSPORT +Specify the type of remote transport to use to manage connections to remote nodes. + +\fBsos collect\fR uses locally installed binaries to connect to and interact with remote +nodes, instead of directly establishing those connections. By default, OpenSSH's ControlPersist +feature is preferred, however certain cluster types may have preferences of their own for how +remote sessions should be established. + +The types of transports supported are currently as follows: + + \fBauto\fR Allow the cluster type to determine the transport used + \fBcontrol_persist\fR Use OpenSSH's ControlPersist feature. This is the default behavior + \fBoc\fR Use a \fBlocally\fR configured \fBoc\fR binary to deploy collection pods on OCP nodes + +.TP \fB\-\-tmp\-dir\fR TMP_DIR Specify a temporary directory to save sos archives to. By default one will be created in /tmp and then removed after sos collect has finished running. diff -Nru sosreport-4.2/man/en/sos-collector.1 sosreport-4.3/man/en/sos-collector.1 --- sosreport-4.2/man/en/sos-collector.1 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sos-collector.1 2022-02-15 04:20:20.000000000 +0000 @@ -11,6 +11,7 @@ [\-\-chroot CHROOT] [\-\-case\-id CASE_ID] [\-\-cluster\-type CLUSTER_TYPE] + [\-\-container\-runtime RUNTIME] [\-e ENABLE_PLUGINS] [--encrypt-key KEY]\fR [--encrypt-pass PASS]\fR @@ -25,7 +26,7 @@ [\-\-nodes NODES] [\-\-no\-pkg\-check] [\-\-no\-local] - [\-\-master MASTER] + [\-\-primary PRIMARY] [\-\-image IMAGE] [\-\-force-pull-image] [\-\-registry-user USER] @@ -43,6 +44,7 @@ [\-\-sos-cmd SOS_CMD] [\-t|\-\-threads THREADS] [\-\-timeout TIMEOUT] + [\-\-transport TRANSPORT] [\-\-tmp\-dir TMP_DIR] [\-v|\-\-verbose] [\-\-verify] @@ -54,7 +56,7 @@ them in a single useful tar archive. sos collect can be run either on a workstation that has SSH key authentication setup -for the nodes in a given cluster, or from a "master" node in a cluster that has SSH +for the nodes in a given cluster, or from a "primary" node in a cluster that has SSH keys configured for the other nodes. Some sosreport options are supported by sos-collect and are passed directly to @@ -99,7 +101,7 @@ \fB\-\-cluster\-type\fR CLUSTER_TYPE When run by itself, sos collect will attempt to identify the type of cluster at play. This is done by checking package or configuration information against the localhost, or -the master node if \fB"--master"\fR is supplied. +the primary node if \fB"--primary"\fR is supplied. Setting \fB--cluster-type\fR skips this step and forcibly sets a particular profile. @@ -112,6 +114,11 @@ to be run, and thus set sosreport options and attempt to determine a list of nodes using that profile. .TP +\fB\-\-container\-runtime\fR RUNTIME +\fB sos report\fR option. Using this with \fBcollect\fR will pass this option thru +to nodes with sos version 4.3 or later. This option controls the default container +runtime plugins will use for collections. See \fBman sos-report\fR. +.TP \fB\-e\fR ENABLE_PLUGINS, \fB\-\-enable\-plugins\fR ENABLE_PLUGINS Sosreport option. Use this to enable a plugin that would otherwise not be run. @@ -152,10 +159,10 @@ \fB\-\-group\fR GROUP Specify an existing host group definition to use. -Host groups are pre-defined settings for the cluster-type, master, and nodes options +Host groups are pre-defined settings for the cluster-type, primary node, and nodes options saved in JSON-formatted files under /var/lib/sos collect/. -If cluster_type and/or master are set in the group, sos collect behaves as if +If cluster_type and/or primary are set in the group, sos collect behaves as if these values were specified on the command-line. If nodes is defined, sos collect \fBextends\fR the \fB\-\-nodes\fR option, if set, @@ -171,7 +178,7 @@ Save the results of this run of sos collect to a host group definition. sos-colllector will write a JSON-formatted file with name GROUP to /var/lib/sos collect/ -with the settings for cluster-type, master, and the node list as discovered by cluster enumeration. +with the settings for cluster-type, primary, and the node list as discovered by cluster enumeration. Note that this means regexes are not directly saved to host groups, but the results of matching against those regexes are. .TP @@ -234,20 +241,20 @@ Do not perform package checks. Most cluster profiles check against installed packages to determine if the cluster profile should be applied or not. -Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the master/local node. +Use this with \fB\-\-cluster-type\fR if there are rpm or apt issues on the primary/local node. .TP \fB\-\-no\-local\fR Do not collect a sosreport from the local system. -If \fB--master\fR is not supplied, it is assumed that the host running sosreport is part of +If \fB--primary\fR is not supplied, it is assumed that the host running sosreport is part of the cluster that is to be collected. Use this option to skip collection of a local sosreport. -This option is NOT needed if \fB--master\fR is provided. +This option is NOT needed if \fB--primary\fR is provided. .TP -\fB\-\-master\fR MASTER -Specify a master node for the cluster. +\fB\-\-primary\fR PRIMARY +Specify a primary node IP address or hostname for the cluster. -If provided, then sos collect will check the master node, not localhost, for determining +If provided, then sos collect will check the primary node, not localhost, for determining the type of cluster in use. .TP \fB\-\-image IMAGE\fR @@ -351,6 +358,21 @@ Default is 180 seconds. .TP +\fB\-\-transport\fR TRANSPORT +Specify the type of remote transport to use to manage connections to remote nodes. + +\fBsos collect\fR uses locally installed binaries to connect to and interact with remote +nodes, instead of directly establishing those connections. By default, OpenSSH's ControlPersist +feature is preferred, however certain cluster types may have preferences of their own for how +remote sessions should be established. + +The types of transports supported are currently as follows: + + \fBauto\fR Allow the cluster type to determine the transport used + \fBcontrol_persist\fR Use OpenSSH's ControlPersist feature. This is the default behavior + \fBoc\fR Use a \fBlocally\fR configured \fBoc\fR binary to deploy collection pods on OCP nodes + +.TP \fB\-\-tmp\-dir\fR TMP_DIR Specify a temporary directory to save sos archives to. By default one will be created in /tmp and then removed after sos collect has finished running. diff -Nru sosreport-4.2/man/en/sos.conf.5 sosreport-4.3/man/en/sos.conf.5 --- sosreport-4.2/man/en/sos.conf.5 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sos.conf.5 2022-02-15 04:20:20.000000000 +0000 @@ -54,7 +54,7 @@ \fBgroups.d\fP This directory is used to store host group configuration files for \fBsos collect\fP. -These files can specify any/all of the \fBmaster\fP, \fBnodes\fP, and \fBcluster-type\fP +These files can specify any/all of the \fBprimary\fP, \fBnodes\fP, and \fBcluster-type\fP options. Users may create their own private host groups in $HOME/.config/sos/groups.d/. If diff -Nru sosreport-4.2/man/en/sos-help.1 sosreport-4.3/man/en/sos-help.1 --- sosreport-4.2/man/en/sos-help.1 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/man/en/sos-help.1 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,41 @@ +.TH SOS HELP 1 "Fri Nov 05 2021" +.SH NAME +sos help - get detailed help information on sos commands and components +.SH SYNOPSIS +.B sos help TOPIC + +.SH DESCRIPTION +\fBsos help\fR is used to retrieve more detailed information on the various SoS +commands and components than is directly available in either other manpages or +--help output. + +This information could for example be investigating a specific plugin to learn more +about its purpose, use case, collections, available plugin options, edge cases, and +more. +.LP +Most aspects of SoS' operation can be investigated this way - the top level functions +such as \fB report, clean,\fR and \fBcollect\fR, as well as constructs that allow those +functions to work; e.g. \fBtransports\fR within \fBsos collect\fR that define how that +function connects to remote nodes. + +.SH REQUIRED ARGUMENTS +.B TOPIC +.TP +The section or topic to retrieve detailed help information for. TOPIC takes the general +form of \fBcommand.component.entity\fR, with \fBcomponent\fR and \fBentity\fR +being optional. +.LP +Top-level \fBcommand\fR help sections will often direct users to \fBcomponent\fR sections +which in turn may point to further \fBentity\fR subsections. + +Some of the more useful or interesting sections are listed below: + + \fBTopic\fR \fBDescription\fR + + \fBreport\fR The \fBsos report\fR command + \fBreport.plugins\fR Information on what report plugins are + \fBreport.plugins.$plugin\fR Information on a specific plugin + \fBclean\fR or \fBmask\fR The \fBsos clean|mask\fR command + \fBcollect\fR The \fBsos collect\fR command + \fBcollect.clusters\fR How \fBcollect\fR enumerates nodes in a cluster + \fBpolicies\fR How SoS behaves on different distributions diff -Nru sosreport-4.2/man/en/sos-mask.1 sosreport-4.3/man/en/sos-mask.1 --- sosreport-4.2/man/en/sos-mask.1 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sos-mask.1 2022-02-15 04:20:20.000000000 +0000 @@ -10,6 +10,7 @@ [\-\-jobs] [\-\-no-update] [\-\-keep-binary-files] + [\-\-archive-type] .SH DESCRIPTION \fBsos clean\fR or \fBsos mask\fR is an sos subcommand used to obfuscate sensitive information from @@ -88,6 +89,31 @@ a third party. Default: False (remove encountered binary files) +.TP +.B \-\-archive-type TYPE +Specify the type of archive that TARGET was generated as. +When sos inspects a TARGET archive, it tries to identify what type of archive it is. +For example, it may be a report generated by \fBsos report\fR, or a collection of those +reports generated by \fBsos collect\fR, which require separate approaches. + +This option may be useful if a given TARGET archive is known to be of a specific type, +but due to unknown reasons or some malformed/missing information in the archive directly, +that is not properly identified by sos. + +The following are accepted values for this option: + + \fBauto\fR Automatically detect the archive type + \fBreport\fR An archive generated by \fBsos report\fR + \fBcollect\fR An archive generated by \fBsos collect\fR + \fBinsights\fR An archive generated by the \fBinsights-client\fR package + +The following may also be used, however note that these do not attempt to pre-load +any information from the archives into the parsers. This means that, among other limitations, +items like host and domain names may not be obfuscated unless an obfuscated mapping already exists +on the system from a previous execution. + + \fBdata-dir\fR A plain directory on the filesystem. + \fBtarball\fR A generic tar archive not associated with any known tool .SH SEE ALSO .BR sos (1) diff -Nru sosreport-4.2/man/en/sos-report.1 sosreport-4.3/man/en/sos-report.1 --- sosreport-4.2/man/en/sos-report.1 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sos-report.1 2022-02-15 04:20:20.000000000 +0000 @@ -14,11 +14,12 @@ [--preset preset] [--add-preset add_preset]\fR [--del-preset del_preset] [--desc description]\fR [--batch] [--build] [--debug] [--dry-run]\fR - [--label label] [--case-id id]\fR + [--estimate-only] [--label label] [--case-id id]\fR [--threads threads]\fR [--plugin-timeout TIMEOUT]\fR [--cmd-timeout TIMEOUT]\fR [--namespaces NAMESPACES]\fR + [--container-runtime RUNTIME]\fR [-s|--sysroot SYSROOT]\fR [-c|--chroot {auto|always|never}\fR [--tmp-dir directory]\fR @@ -299,6 +300,24 @@ Note that specific plugins may provide a similar `namespaces` plugin option. If the plugin option is used, it will override this option. +.TP +.B \--container-runtime RUNTIME +Force the use of the specified RUNTIME as the default runtime that plugins will +use to collect data from and about containers and container images. By default, +the setting of \fBauto\fR results in the local policy determining what runtime +will be the default runtime (in configurations where multiple runtimes are installed +and active). + +If no container runtimes are active, this option is ignored. If there are runtimes +active, but not one with a name matching RUNTIME, sos will abort. + +Setting this to \fBnone\fR, \fBoff\fR, or \fBdisabled\fR will cause plugins to +\fBNOT\fR leverage any active runtimes for collections. Note that if disabled, plugins +specifically for runtimes (e.g. the podman or docker plugins) will still collect +general data about the runtime, but will not inspect existing containers or images. + +Default: 'auto' (policy determined) +.TP .B \--case-id NUMBER Specify a case identifier to associate with the archive. Identifiers may include alphanumeric characters, commas and periods ('.'). @@ -317,6 +336,21 @@ to understand the actions that sos would have taken without the dry run option. .TP +.B \--estimate-only +Estimate disk space requirements when running sos report. This can be valuable +to prevent sosreport working dir to consume all free disk space. No plugin data +is available at the end. + +Plugins will be collected sequentially, size of collected files and commands outputs +will be calculated and the plugin files will be immediatelly deleted prior execution +of the next plugin. This still can consume whole free disk space, though. + +Please note, size estimations may not be accurate for highly utilized systems due to +changes between an estimate and a real execution. Also some difference between +estimation (using `stat` command) and other commands used (i.e. `du`). + +A rule of thumb is to reserve at least double the estimation. +.TP .B \--upload If specified, attempt to upload the resulting archive to a vendor defined location. diff -Nru sosreport-4.2/man/en/sosreport.1 sosreport-4.3/man/en/sosreport.1 --- sosreport-4.2/man/en/sosreport.1 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/man/en/sosreport.1 2022-02-15 04:20:20.000000000 +0000 @@ -14,11 +14,12 @@ [--preset preset] [--add-preset add_preset]\fR [--del-preset del_preset] [--desc description]\fR [--batch] [--build] [--debug] [--dry-run]\fR - [--label label] [--case-id id]\fR + [--estimate-only] [--label label] [--case-id id]\fR [--threads threads]\fR [--plugin-timeout TIMEOUT]\fR [--cmd-timeout TIMEOUT]\fR [--namespaces NAMESPACES]\fR + [--container-runtime RUNTIME]\fR [-s|--sysroot SYSROOT]\fR [-c|--chroot {auto|always|never}\fR [--tmp-dir directory]\fR @@ -299,6 +300,24 @@ Note that specific plugins may provide a similar `namespaces` plugin option. If the plugin option is used, it will override this option. +.TP +.B \--container-runtime RUNTIME +Force the use of the specified RUNTIME as the default runtime that plugins will +use to collect data from and about containers and container images. By default, +the setting of \fBauto\fR results in the local policy determining what runtime +will be the default runtime (in configurations where multiple runtimes are installed +and active). + +If no container runtimes are active, this option is ignored. If there are runtimes +active, but not one with a name matching RUNTIME, sos will abort. + +Setting this to \fBnone\fR, \fBoff\fR, or \fBdisabled\fR will cause plugins to +\fBNOT\fR leverage any active runtimes for collections. Note that if disabled, plugins +specifically for runtimes (e.g. the podman or docker plugins) will still collect +general data about the runtime, but will not inspect existing containers or images. + +Default: 'auto' (policy determined) +.TP .B \--case-id NUMBER Specify a case identifier to associate with the archive. Identifiers may include alphanumeric characters, commas and periods ('.'). @@ -317,6 +336,21 @@ to understand the actions that sos would have taken without the dry run option. .TP +.B \--estimate-only +Estimate disk space requirements when running sos report. This can be valuable +to prevent sosreport working dir to consume all free disk space. No plugin data +is available at the end. + +Plugins will be collected sequentially, size of collected files and commands outputs +will be calculated and the plugin files will be immediatelly deleted prior execution +of the next plugin. This still can consume whole free disk space, though. + +Please note, size estimations may not be accurate for highly utilized systems due to +changes between an estimate and a real execution. Also some difference between +estimation (using `stat` command) and other commands used (i.e. `du`). + +A rule of thumb is to reserve at least double the estimation. +.TP .B \--upload If specified, attempt to upload the resulting archive to a vendor defined location. diff -Nru sosreport-4.2/plugins_overview.py sosreport-4.3/plugins_overview.py --- sosreport-4.2/plugins_overview.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/plugins_overview.py 2022-02-15 04:20:20.000000000 +0000 @@ -73,7 +73,7 @@ # if plugname != 'bcache': # continue plugs_data[plugname] = { - 'sourcecode': 'https://github.com/sosreport/sos/blob/master/sos/report/plugins/%s.py' % plugname, + 'sourcecode': 'https://github.com/sosreport/sos/blob/main/sos/report/plugins/%s.py' % plugname, 'distros': [], 'profiles': [], 'packages': [], diff -Nru sosreport-4.2/pylintrc sosreport-4.3/pylintrc --- sosreport-4.2/pylintrc 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/pylintrc 1970-01-01 00:00:00.000000000 +0000 @@ -1,354 +0,0 @@ -# lint Python modules using external checkers. -# -# This is the main checker controling the other ones and the reports -# generation. It is itself both a raw checker and an astng checker in order -# to: -# * handle message activation / deactivation at the module level -# * handle some basic but necessary stats'data (number of classes, methods...) -# -# This checker also defines the following reports: -# * R0001: Total errors / warnings -# * R0002: % errors / warnings by module -# * R0003: Messages -# * R0004: Global evaluation -[MASTER] - -# Profiled execution. -profile=no - -# Add to the black list. It should be a base name, not a -# path. You may set this option multiple times. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=yes - -# Set the cache size for astng objects. -cache-size=500 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - -[REPORTS] - -# Tells wether to display a full report or only the messages -reports=yes - -# Use HTML as output format instead of text -html=no - -# Use a parseable text output format, so your favorite text editor will be able -# to jump to the line corresponding to a message. -parseable=yes - -# Colorizes text output using ansi escape codes -color=no - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Python expression which should return a note less than 10 (10 is the highest -# note).You have access to the variables errors warning, statement which -# respectivly contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (R0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (R0004). -comment=no - -# Include message's id in output -include-ids=yes - - -# checks for -# * unused variables / imports -# * undefined variables -# * redefinition of variable from builtins or from an outer scope -# * use of variable before assigment -# -[VARIABLES] - -# Enable / disable this checker -enable-variables=yes - -# Tells wether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching names used for dummy variables (i.e. not used). -dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins=_ - - -# try to find bugs in the code using type inference -# -[TYPECHECK] - -# Enable / disable this checker -enable-typecheck=yes - -# Tells wether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# When zope mode is activated, consider the acquired-members option to ignore -# access to some undefined attributes. -zope=no - -# List of members which are usually get through zope's acquisition mecanism and -# so shouldn't trigger E0201 when accessed (need zope=yes to be considered. -acquired-members=REQUEST,acl_users,aq_parent - - -# checks for : -# * doc strings -# * modules / classes / functions / methods / arguments / variables name -# * number of arguments, local variables, branchs, returns and statements in -# functions, methods -# * required module attributes -# * dangerous default values as arguments -# * redefinition of function / method / class -# * uses of the global statement -# -# This checker also defines the following reports: -# * R0101: Statistics by type -[BASIC] - -# Enable / disable this checker -enable-basic=yes - -#disable-msg=C0121 - -# Required attributes for module, separated by a comma -required-attributes= - -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__ - -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression which should only match correct module level names -const-rgx=(([A-Z_][A-Z1-9_]*)|(__.*__))$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct method names -method-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][A-Za-z0-9_]{0,30}$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input - - -# checks for sign of poor/misdesign: -# * number of methods, attributes, local variables... -# * size, complexity of functions, methods -# -[DESIGN] - -# Enable / disable this checker -enable-design=yes - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branchs=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -# checks for : -# * methods without self as first argument -# * overriden methods signature -# * access only to existant members via self -# * attributes not defined in the __init__ method -# * supported interfaces implementation -# * unreachable code -# -[CLASSES] - -# Enable / disable this checker -enable-classes=yes - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - - -# checks for -# * external modules dependencies -# * relative / wildcard imports -# * cyclic imports -# * uses of deprecated modules -# -# This checker also defines the following reports: -# * R0401: External dependencies -# * R0402: Modules dependencies graph -[IMPORTS] - -# Enable / disable this checker -enable-imports=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report R0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report R0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report R0402 must -# not be disabled) -int-import-graph= - - -# checks for usage of new style capabilities on old style classes and -# other new/old styles conflicts problems -# * use of property, __slots__, super -# * "super" usage -# * raising a new style class as exception -# -[NEWSTYLE] - -# Enable / disable this checker -enable-newstyle=yes - - -# checks for -# * excepts without exception filter -# * string exceptions -# -[EXCEPTIONS] - -# Enable / disable this checker -enable-exceptions=yes - - -# checks for : -# * unauthorized constructions -# * strict indentation -# * line length -# * use of <> instead of != -# -[FORMAT] - -# Enable / disable this checker -enable-format=yes - -# Maximum number of characters on a single line. -max-line-length=132 - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - - -# checks for similarities and duplicated code. This computation may be -# memory / CPU intensive, so you should disable it if you experiments some -# problems. -# -# This checker also defines the following reports: -# * R0801: Duplication -[SIMILARITIES] - -# Enable / disable this checker -enable-similarities=yes - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - - -# checks for: -# * warning notes in the code like FIXME, XXX -# * PEP 263: source code with non ascii character but no encoding declaration -# -[MISCELLANEOUS] - -# Enable / disable this checker -enable-miscellaneous=yes - -# List of note tags to take in consideration, separated by a comma. Default to -# FIXME, XXX, TODO -notes=FIXME,XXX,TODO - - -# does not check anything but gives some raw metrics : -# * total number of lines -# * total number of code lines -# * total number of docstring lines -# * total number of comments lines -# * total number of empty lines -# -# This checker also defines the following reports: -# * R0701: Raw metrics -[METRICS] - -# Enable / disable this checker -enable-metrics=no diff -Nru sosreport-4.2/README.md sosreport-4.3/README.md --- sosreport-4.2/README.md 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/README.md 2022-02-15 04:20:20.000000000 +0000 @@ -1,4 +1,4 @@ -[![Build Status](https://api.cirrus-ci.com/github/sosreport/sos.svg?branch=master)](https://cirrus-ci.com/github/sosreport/sos) [![Documentation Status](https://readthedocs.org/projects/sos/badge/?version=master)](https://sos.readthedocs.io/en/master/?badge=master) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/sosreport/sos.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/sosreport/sos/context:python) +[![Build Status](https://api.cirrus-ci.com/github/sosreport/sos.svg?branch=main)](https://cirrus-ci.com/github/sosreport/sos) [![Documentation Status](https://readthedocs.org/projects/sos/badge/?version=main)](https://sos.readthedocs.io/en/main/?badge=main) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/sosreport/sos.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/sosreport/sos/context:python) # SoS @@ -12,11 +12,12 @@ For the latest version, to contribute, and for more information, please visit the project pages or join the mailing list. -To clone the current master (development) branch run: +To clone the current main (development) branch run: ``` git clone git://github.com/sosreport/sos.git ``` + ## Reporting bugs Please report bugs via the mailing list or by opening an issue in the [GitHub @@ -27,6 +28,7 @@ The SoS project has rooms in Matrix and in Libera.Chat. Matrix Room: #sosreport:matrix.org + Libera.Chat: #sos These rooms are bridged, so joining either is sufficient as messages from either will @@ -36,19 +38,36 @@ ## Mailing list -The [sos-devel][4] is the mailing list for any sos-related questions and +The [sos-devel][4] list is the mailing list for any sos-related questions and discussion. Patch submissions and reviews are welcome too. ## Patches and pull requests Patches can be submitted via the mailing list or as GitHub pull requests. If -using GitHub please make sure your branch applies to the current master as a +using GitHub please make sure your branch applies to the current main branch as a 'fast forward' merge (i.e. without creating a merge commit). Use the `git -rebase` command to update your branch to the current master if necessary. +rebase` command to update your branch to the current main if necessary. Please refer to the [contributor guidelines][0] for guidance on formatting patches and commit messages. +Before sending a [pull request][0], it is advisable to check your contribution +against the `flake8` linter, the unit tests, and the stage one avocado test suite: + +``` +# from within the git checkout +$ flake8 sos +$ nosetests -v tests/unittests/ + +# as root +# PYTHONPATH=tests/ avocado run --test-runner=runner -t stageone tests/{cleaner,collect,report,vendor}_tests +``` + +Note that the avocado test suite will generate and remove several reports over its +execution, but no changes will be made to your local system. + +All contributions must pass the entire test suite before being accepted. + ## Documentation User and API [documentation][6] is automatically generated using [Sphinx][7] @@ -66,12 +85,15 @@ python3 setup.py build_sphinx -a ``` -Please run `./tests/simple.sh` before sending a [pull request][0], and run the -test suite manually using the `nosetests` command (ideally for the -set of Python versions currently supported by `sos` upstream). ### Wiki +For more in-depth information on the project's features and functionality, please +see [the GitHub wiki][9]. + +If you are interested in contributing an entirely new plugin, or extending sos to +support your distribution of choice, please see these wiki pages: + * [How to write a plugin][1] * [How to write a policy][2] * [Plugin options][3] @@ -94,9 +116,9 @@ $ sudo ./bin/sosreport ``` -If you want to run it with all the options enabled (this can take a long time) +To see a list of all available plugins and plugin options, run ``` -$ sudo ./bin/sos report -a +$ sudo ./bin/sos report -l ``` @@ -111,20 +133,20 @@ Fedora/RHEL users install via yum: ``` -yum install sos +# yum install sos ``` Debian users install via apt: ``` -apt install sosreport +# apt install sosreport ``` Ubuntu (14.04 LTS and above) users install via apt: ``` -sudo apt install sosreport +# sudo apt install sosreport ``` [0]: https://github.com/sosreport/sos/wiki/Contribution-Guidelines @@ -136,3 +158,4 @@ [6]: https://sos.readthedocs.org/ [7]: https://www.sphinx-doc.org/ [8]: https://www.readthedocs.org/ + [9]: https://github.com/sosreport/sos/wiki diff -Nru sosreport-4.2/setup.py sosreport-4.3/setup.py --- sosreport-4.2/setup.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/setup.py 2022-02-15 04:20:20.000000000 +0000 @@ -101,8 +101,8 @@ 'sos.policies.distros', 'sos.policies.runtimes', 'sos.policies.package_managers', 'sos.policies.init_systems', 'sos.report', 'sos.report.plugins', 'sos.collector', - 'sos.collector.clusters', 'sos.cleaner', 'sos.cleaner.mappings', - 'sos.cleaner.parsers' + 'sos.collector.clusters', 'sos.collector.transports', 'sos.cleaner', + 'sos.cleaner.mappings', 'sos.cleaner.parsers', 'sos.cleaner.archives' ], cmdclass=cmdclass, command_options=command_options, diff -Nru sosreport-4.2/snap/snapcraft.yaml sosreport-4.3/snap/snapcraft.yaml --- sosreport-4.2/snap/snapcraft.yaml 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/snap/snapcraft.yaml 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,37 @@ +name: sosreport +summary: Sos is an extensible, portable, support data collection tool +description: | + Sos is an extensible, portable, support data collection tool + primarily aimed at Linux distributions and other UNIX-like operating + systems. +grade: stable +base: core20 +confinement: classic +adopt-info: sos + +parts: + sos: + plugin: python + source: . + override-pull: | + snapcraftctl pull + snapcraftctl set-version $(git describe --tags --always) + build-packages: + - git + - python3 + - snapcraft + - gettext + +apps: + sos: + environment: + PYTHONPATH: ${PYTHONPATH}:${SNAP}/lib/python3.8 + command: bin/sos + sosreport: + environment: + PYTHONPATH: ${PYTHONPATH}:${SNAP}/lib/python3.8 + command: bin/sos report + sos-collector: + environment: + PYTHONPATH: ${PYTHONPATH}:${SNAP}/lib/python3.8 + command: bin/sos collector diff -Nru sosreport-4.2/sos/archive.py sosreport-4.3/sos/archive.py --- sosreport-4.2/sos/archive.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/archive.py 2022-02-15 04:20:20.000000000 +0000 @@ -153,7 +153,7 @@ return (os.path.join(self._archive_root, name)) def join_sysroot(self, path): - if path.startswith(self.sysroot): + if not self.sysroot or path.startswith(self.sysroot): return path if path[0] == os.sep: path = path[1:] @@ -251,7 +251,7 @@ return dest - def _check_path(self, src, path_type, dest=None, force=False): + def check_path(self, src, path_type, dest=None, force=False): """Check a new destination path in the archive. Since it is possible for multiple plugins to collect the same @@ -345,7 +345,7 @@ if not dest: dest = src - dest = self._check_path(dest, P_FILE) + dest = self.check_path(dest, P_FILE) if not dest: return @@ -384,7 +384,7 @@ # over any exixting content in the archive, since it is used by # the Plugin postprocessing hooks to perform regex substitution # on file content. - dest = self._check_path(dest, P_FILE, force=True) + dest = self.check_path(dest, P_FILE, force=True) f = codecs.open(dest, mode, encoding='utf-8') if isinstance(content, bytes): @@ -397,7 +397,7 @@ def add_binary(self, content, dest): with self._path_lock: - dest = self._check_path(dest, P_FILE) + dest = self.check_path(dest, P_FILE) if not dest: return @@ -409,7 +409,7 @@ def add_link(self, source, link_name): self.log_debug("adding symlink at '%s' -> '%s'" % (link_name, source)) with self._path_lock: - dest = self._check_path(link_name, P_LINK) + dest = self.check_path(link_name, P_LINK) if not dest: return @@ -484,10 +484,10 @@ """ # Establish path structure with self._path_lock: - self._check_path(path, P_DIR) + self.check_path(path, P_DIR) def add_node(self, path, mode, device): - dest = self._check_path(path, P_NODE) + dest = self.check_path(path, P_NODE) if not dest: return diff -Nru sosreport-4.2/sos/cleaner/archives/generic.py sosreport-4.3/sos/cleaner/archives/generic.py --- sosreport-4.2/sos/cleaner/archives/generic.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/cleaner/archives/generic.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,52 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos.cleaner.archives import SoSObfuscationArchive + +import os +import tarfile + + +class DataDirArchive(SoSObfuscationArchive): + """A plain directory on the filesystem that is not directly associated with + any known or supported collection utility + """ + + type_name = 'data_dir' + description = 'unassociated directory' + + @classmethod + def check_is_type(cls, arc_path): + return os.path.isdir(arc_path) + + def set_archive_root(self): + return os.path.abspath(self.archive_path) + + +class TarballArchive(SoSObfuscationArchive): + """A generic tar archive that is not associated with any known or supported + collection utility + """ + + type_name = 'tarball' + description = 'unassociated tarball' + + @classmethod + def check_is_type(cls, arc_path): + try: + return tarfile.is_tarfile(arc_path) + except Exception: + return False + + def set_archive_root(self): + if self.tarobj.firstmember.isdir(): + return self.tarobj.firstmember.name + return '' diff -Nru sosreport-4.2/sos/cleaner/archives/__init__.py sosreport-4.3/sos/cleaner/archives/__init__.py --- sosreport-4.2/sos/cleaner/archives/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/cleaner/archives/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,395 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import logging +import os +import shutil +import stat +import tarfile +import re + +from concurrent.futures import ProcessPoolExecutor + + +# python older than 3.8 will hit a pickling error when we go to spawn a new +# process for extraction if this method is a part of the SoSObfuscationArchive +# class. So, the simplest solution is to remove it from the class. +def extract_archive(archive_path, tmpdir): + archive = tarfile.open(archive_path) + path = os.path.join(tmpdir, 'cleaner') + archive.extractall(path) + archive.close() + return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0]) + + +class SoSObfuscationArchive(): + """A representation of an extracted archive or an sos archive build + directory which is used by SoSCleaner. + + Each archive that needs to be obfuscated is loaded into an instance of this + class. All report-level operations should be contained within this class. + """ + + file_sub_list = [] + total_sub_count = 0 + removed_file_count = 0 + type_name = 'undetermined' + description = 'undetermined' + is_nested = False + skip_files = [] + prep_files = {} + + def __init__(self, archive_path, tmpdir): + self.archive_path = archive_path + self.final_archive_path = self.archive_path + self.tmpdir = tmpdir + self.archive_name = self.archive_path.split('/')[-1].split('.tar')[0] + self.ui_name = self.archive_name + self.soslog = logging.getLogger('sos') + self.ui_log = logging.getLogger('sos_ui') + self.skip_list = self._load_skip_list() + self.is_extracted = False + self._load_self() + self.archive_root = '' + self.log_info( + "Loaded %s as type %s" + % (self.archive_path, self.description) + ) + + @classmethod + def check_is_type(cls, arc_path): + """Check if the archive is a well-known type we directly support""" + return False + + def _load_self(self): + if self.is_tarfile: + self.tarobj = tarfile.open(self.archive_path) + + def get_nested_archives(self): + """Return a list of ObfuscationArchives that represent additional + archives found within the target archive. For example, an archive from + `sos collect` will return a list of ``SoSReportArchive`` objects. + + This should be overridden by individual types of ObfuscationArchive's + """ + return [] + + def get_archive_root(self): + """Set the root path for the archive that should be prepended to any + filenames given to methods in this class. + """ + if self.is_tarfile: + toplevel = self.tarobj.firstmember + if toplevel.isdir(): + return toplevel.name + else: + return os.sep + return os.path.abspath(self.archive_path) + + def report_msg(self, msg): + """Helper to easily format ui messages on a per-report basis""" + self.ui_log.info("{:<50} {}".format(self.ui_name + ' :', msg)) + + def _fmt_log_msg(self, msg): + return "[cleaner:%s] %s" % (self.archive_name, msg) + + def log_debug(self, msg): + self.soslog.debug(self._fmt_log_msg(msg)) + + def log_info(self, msg): + self.soslog.info(self._fmt_log_msg(msg)) + + def _load_skip_list(self): + """Provide a list of files and file regexes to skip obfuscation on + + Returns: list of files and file regexes + """ + return [ + 'proc/kallsyms', + 'sosreport-', + 'sys/firmware', + 'sys/fs', + 'sys/kernel/debug', + 'sys/module' + ] + + @property + def is_tarfile(self): + try: + return tarfile.is_tarfile(self.archive_path) + except Exception: + return False + + def remove_file(self, fname): + """Remove a file from the archive. This is used when cleaner encounters + a binary file, which we cannot reliably obfuscate. + """ + full_fname = self.get_file_path(fname) + # don't call a blank remove() here + if full_fname: + self.log_info("Removing binary file '%s' from archive" % fname) + os.remove(full_fname) + self.removed_file_count += 1 + + def format_file_name(self, fname): + """Based on the type of archive we're dealing with, do whatever that + archive requires to a provided **relative** filepath to be able to + access it within the archive + """ + if not self.is_extracted: + if not self.archive_root: + self.archive_root = self.get_archive_root() + return os.path.join(self.archive_root, fname) + else: + return os.path.join(self.extracted_path, fname) + + def get_file_content(self, fname): + """Return the content from the specified fname. Particularly useful for + tarball-type archives so we can retrieve prep file contents prior to + extracting the entire archive + """ + if self.is_extracted is False and self.is_tarfile: + filename = self.format_file_name(fname) + try: + return self.tarobj.extractfile(filename).read().decode('utf-8') + except KeyError: + self.log_debug( + "Unable to retrieve %s: no such file in archive" % fname + ) + return '' + else: + with open(self.format_file_name(fname), 'r') as to_read: + return to_read.read() + + def extract(self, quiet=False): + if self.is_tarfile: + if not quiet: + self.report_msg("Extracting...") + self.extracted_path = self.extract_self() + self.is_extracted = True + else: + self.extracted_path = self.archive_path + # if we're running as non-root (e.g. collector), then we can have a + # situation where a particular path has insufficient permissions for + # us to rewrite the contents and/or add it to the ending tarfile. + # Unfortunately our only choice here is to change the permissions + # that were preserved during report collection + if os.getuid() != 0: + self.log_debug('Verifying permissions of archive contents') + for dirname, dirs, files in os.walk(self.extracted_path): + try: + for _dir in dirs: + _dirname = os.path.join(dirname, _dir) + _dir_perms = os.stat(_dirname).st_mode + os.chmod(_dirname, _dir_perms | stat.S_IRWXU) + for filename in files: + fname = os.path.join(dirname, filename) + # protect against symlink race conditions + if not os.path.exists(fname) or os.path.islink(fname): + continue + if (not os.access(fname, os.R_OK) or not + os.access(fname, os.W_OK)): + self.log_debug( + "Adding owner rw permissions to %s" + % fname.split(self.archive_path)[-1] + ) + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR) + except Exception as err: + self.log_debug("Error while trying to set perms: %s" % err) + self.log_debug("Extracted path is %s" % self.extracted_path) + + def rename_top_dir(self, new_name): + """Rename the top-level directory to new_name, which should be an + obfuscated string that scrubs the hostname from the top-level dir + which would be named after the unobfuscated sos report + """ + _path = self.extracted_path.replace(self.archive_name, new_name) + self.archive_name = new_name + os.rename(self.extracted_path, _path) + self.extracted_path = _path + + def get_compression(self): + """Return the compression type used by the archive, if any. This is + then used by SoSCleaner to generate a policy-derived compression + command to repack the archive + """ + if self.is_tarfile: + if self.archive_path.endswith('xz'): + return 'xz' + return 'gz' + return None + + def build_tar_file(self, method): + """Pack the extracted archive as a tarfile to then be re-compressed + """ + mode = 'w' + tarpath = self.extracted_path + '-obfuscated.tar' + compr_args = {} + if method: + mode += ":%s" % method + tarpath += ".%s" % method + if method == 'xz': + compr_args = {'preset': 3} + else: + compr_args = {'compresslevel': 6} + self.log_debug("Building tar file %s" % tarpath) + tar = tarfile.open(tarpath, mode=mode, **compr_args) + tar.add(self.extracted_path, + arcname=os.path.split(self.archive_name)[1]) + tar.close() + return tarpath + + def compress(self, method): + """Execute the compression command, and set the appropriate final + archive path for later reference by SoSCleaner on a per-archive basis + """ + try: + self.final_archive_path = self.build_tar_file(method) + except Exception as err: + self.log_debug("Exception while re-compressing archive: %s" % err) + raise + self.log_debug("Compressed to %s" % self.final_archive_path) + try: + self.remove_extracted_path() + except Exception as err: + self.log_debug("Failed to remove extraction directory: %s" % err) + self.report_msg('Failed to remove temporary extraction directory') + + def remove_extracted_path(self): + """After the tarball has been re-compressed, remove the extracted path + so that we don't take up that duplicate space any longer during + execution + """ + def force_delete_file(action, name, exc): + os.chmod(name, stat.S_IWUSR) + if os.path.isfile(name): + os.remove(name) + else: + shutil.rmtree(name) + self.log_debug("Removing %s" % self.extracted_path) + shutil.rmtree(self.extracted_path, onerror=force_delete_file) + + def extract_self(self): + """Extract an archive into our tmpdir so that we may inspect it or + iterate through its contents for obfuscation + """ + + with ProcessPoolExecutor(1) as _pool: + _path_future = _pool.submit(extract_archive, + self.archive_path, self.tmpdir) + path = _path_future.result() + return path + + def get_file_list(self): + """Return a list of all files within the archive""" + self.file_list = [] + for dirname, dirs, files in os.walk(self.extracted_path): + for _dir in dirs: + _dirpath = os.path.join(dirname, _dir) + # catch dir-level symlinks + if os.path.islink(_dirpath) and os.path.isdir(_dirpath): + self.file_list.append(_dirpath) + for filename in files: + self.file_list.append(os.path.join(dirname, filename)) + return self.file_list + + def get_directory_list(self): + """Return a list of all directories within the archive""" + dir_list = [] + for dirname, dirs, files in os.walk(self.extracted_path): + dir_list.append(dirname) + return dir_list + + def update_sub_count(self, fname, count): + """Called when a file has finished being parsed and used to track + total substitutions made and number of files that had changes made + """ + self.file_sub_list.append(fname) + self.total_sub_count += count + + def get_file_path(self, fname): + """Return the filepath of a specific file within the archive so that + it may be selectively inspected if it exists + """ + _path = os.path.join(self.extracted_path, fname.lstrip('/')) + return _path if os.path.exists(_path) else '' + + def should_skip_file(self, filename): + """Checks the provided filename against a list of filepaths to not + perform obfuscation on, as defined in self.skip_list + + Positional arguments: + + :param filename str: Filename relative to the extracted + archive root + """ + + if (not os.path.isfile(self.get_file_path(filename)) and not + os.path.islink(self.get_file_path(filename))): + return True + + for _skip in self.skip_list: + if filename.startswith(_skip) or re.match(_skip, filename): + return True + return False + + def should_remove_file(self, fname): + """Determine if the file should be removed or not, due to an inability + to reliably obfuscate that file based on the filename. + + :param fname: Filename relative to the extracted archive root + :type fname: ``str`` + + :returns: ``True`` if the file cannot be reliably obfuscated + :rtype: ``bool`` + """ + obvious_removes = [ + r'.*\.gz', # TODO: support flat gz/xz extraction + r'.*\.xz', + r'.*\.bzip2', + r'.*\.tar\..*', # TODO: support archive unpacking + r'.*\.txz$', + r'.*\.tgz$', + r'.*\.bin', + r'.*\.journal', + r'.*\~$' + ] + + # if the filename matches, it is obvious we can remove them without + # doing the read test + for _arc_reg in obvious_removes: + if re.match(_arc_reg, fname): + return True + + if os.path.isfile(self.get_file_path(fname)): + return self.file_is_binary(fname) + # don't fail on dir-level symlinks + return False + + def file_is_binary(self, fname): + """Determine if the file is a binary file or not. + + + :param fname: Filename relative to the extracted archive root + :type fname: ``str`` + + :returns: ``True`` if file is binary, else ``False`` + :rtype: ``bool`` + """ + with open(self.get_file_path(fname), 'tr') as tfile: + try: + # when opened as above (tr), reading binary content will raise + # an exception + tfile.read(1) + return False + except UnicodeDecodeError: + return True + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/cleaner/archives/insights.py sosreport-4.3/sos/cleaner/archives/insights.py --- sosreport-4.2/sos/cleaner/archives/insights.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/cleaner/archives/insights.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,42 @@ +# Copyright 2021 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos.cleaner.archives import SoSObfuscationArchive + +import tarfile + + +class InsightsArchive(SoSObfuscationArchive): + """This class represents archives generated by the insights-client utility + for RHEL systems. + """ + + type_name = 'insights' + description = 'insights-client archive' + + prep_files = { + 'hostname': 'data/insights_commands/hostname_-f', + 'ip': 'data/insights_commands/ip_addr', + 'mac': 'data/insights_commands/ip_addr' + } + + @classmethod + def check_is_type(cls, arc_path): + try: + return tarfile.is_tarfile(arc_path) and 'insights-' in arc_path + except Exception: + return False + + def get_archive_root(self): + top = self.archive_path.split('/')[-1].split('.tar')[0] + if self.tarobj.firstmember.name == '.': + top = './' + top + return top diff -Nru sosreport-4.2/sos/cleaner/archives/sos.py sosreport-4.3/sos/cleaner/archives/sos.py --- sosreport-4.2/sos/cleaner/archives/sos.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/cleaner/archives/sos.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,111 @@ +# Copyright 2021 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos.cleaner.archives import SoSObfuscationArchive + +import os +import tarfile + + +class SoSReportArchive(SoSObfuscationArchive): + """This is the class representing an sos report, or in other words the + type the archive the SoS project natively generates + """ + + type_name = 'report' + description = 'sos report archive' + prep_files = { + 'hostname': [ + 'sos_commands/host/hostname', + 'etc/hosts' + ], + 'ip': 'sos_commands/networking/ip_-o_addr', + 'mac': 'sos_commands/networking/ip_-d_address', + 'username': [ + 'sos_commands/login/lastlog_-u_1000-60000', + 'sos_commands/login/lastlog_-u_60001-65536', + 'sos_commands/login/lastlog_-u_65537-4294967295', + # AD users will be reported here, but favor the lastlog files since + # those will include local users who have not logged in + 'sos_commands/login/last', + 'etc/cron.allow', + 'etc/cron.deny' + ] + } + + @classmethod + def check_is_type(cls, arc_path): + try: + return tarfile.is_tarfile(arc_path) and 'sosreport-' in arc_path + except Exception: + return False + + +class SoSReportDirectory(SoSReportArchive): + """This is the archive class representing a build directory, or in other + words what `sos report --clean` will end up using for in-line obfuscation + """ + + type_name = 'report_dir' + description = 'sos report directory' + + @classmethod + def check_is_type(cls, arc_path): + if os.path.isdir(arc_path): + return 'sos_logs' in os.listdir(arc_path) + return False + + +class SoSCollectorArchive(SoSObfuscationArchive): + """Archive class representing the tarball created by ``sos collect``. It + will not provide prep files on its own, however it will provide a list + of SoSReportArchive's which will then be used to prep the parsers + """ + + type_name = 'collect' + description = 'sos collect tarball' + is_nested = True + + @classmethod + def check_is_type(cls, arc_path): + try: + return (tarfile.is_tarfile(arc_path) and 'sos-collect' in arc_path) + except Exception: + return False + + def get_nested_archives(self): + self.extract(quiet=True) + _path = self.extracted_path + archives = [] + for fname in os.listdir(_path): + arc_name = os.path.join(_path, fname) + if 'sosreport-' in fname and tarfile.is_tarfile(arc_name): + archives.append(SoSReportArchive(arc_name, self.tmpdir)) + return archives + + +class SoSCollectorDirectory(SoSCollectorArchive): + """The archive class representing the temp directory used by ``sos + collect`` when ``--clean`` is used during runtime. + """ + + type_name = 'collect_dir' + description = 'sos collect directory' + + @classmethod + def check_is_type(cls, arc_path): + if os.path.isdir(arc_path): + for fname in os.listdir(arc_path): + if 'sos-collector-' in fname: + return True + return False + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/cleaner/__init__.py sosreport-4.3/sos/cleaner/__init__.py --- sosreport-4.2/sos/cleaner/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -14,7 +14,6 @@ import os import re import shutil -import tarfile import tempfile from concurrent.futures import ThreadPoolExecutor @@ -27,20 +26,55 @@ from sos.cleaner.parsers.hostname_parser import SoSHostnameParser from sos.cleaner.parsers.keyword_parser import SoSKeywordParser from sos.cleaner.parsers.username_parser import SoSUsernameParser -from sos.cleaner.obfuscation_archive import SoSObfuscationArchive +from sos.cleaner.archives.sos import (SoSReportArchive, SoSReportDirectory, + SoSCollectorArchive, + SoSCollectorDirectory) +from sos.cleaner.archives.generic import DataDirArchive, TarballArchive +from sos.cleaner.archives.insights import InsightsArchive from sos.utilities import get_human_readable from textwrap import fill class SoSCleaner(SoSComponent): - """Take an sos report, or collection of sos reports, and scrub them of - potentially sensitive data such as IP addresses, hostnames, MAC addresses, - etc.. that are not obfuscated by individual plugins + """ + This function is designed to obfuscate potentially sensitive information + from an sos report archive in a consistent and reproducible manner. + + It may either be invoked during the creation of a report by using the + --clean option in the report command, or may be used on an already existing + archive by way of 'sos clean'. + + The target of obfuscation are items such as IP addresses, MAC addresses, + hostnames, usernames, and also keywords provided by users via the + --keywords and/or --keyword-file options. + + For every collection made in a report the collection is parsed for such + items, and when items are found SoS will generate an obfuscated replacement + for it, and in all places that item is found replace the text with the + obfuscated replacement mapped to it. These mappings are saved locally so + that future iterations will maintain the same consistent obfuscation + pairing. + + In the case of IP addresses, support is for IPv4 and efforts are made to + keep network topology intact so that later analysis is as accurate and + easily understandable as possible. If an IP address is encountered that we + cannot determine the netmask for, a random IP address is used instead. + + For hostnames, domains are obfuscated as whole units, leaving the TLD in + place. + + For instance, 'example.com' may be obfuscated to 'obfuscateddomain0.com' + and 'foo.example.com' may end up being 'obfuscateddomain1.com'. + + Users will be notified of a 'mapping' file that records all items and the + obfuscated counterpart mapped to them for ease of reference later on. This + file should be kept private. """ desc = "Obfuscate sensitive networking information in a report" arg_defaults = { + 'archive_type': 'auto', 'domains': [], 'jobs': 4, 'keywords': [], @@ -70,6 +104,7 @@ self.from_cmdline = False if not hasattr(self.opts, 'jobs'): self.opts.jobs = 4 + self.opts.archive_type = 'auto' self.soslog = logging.getLogger('sos') self.ui_log = logging.getLogger('sos_ui') # create the tmp subdir here to avoid a potential race condition @@ -92,6 +127,18 @@ SoSUsernameParser(self.cleaner_mapping, self.opts.usernames) ] + self.archive_types = [ + SoSReportDirectory, + SoSReportArchive, + SoSCollectorDirectory, + SoSCollectorArchive, + InsightsArchive, + # make sure these two are always last as they are fallbacks + DataDirArchive, + TarballArchive + ] + self.nested_archive = None + self.log_info("Cleaner initialized. From cmdline: %s" % self.from_cmdline) @@ -114,6 +161,11 @@ _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' return _fmt + @classmethod + def display_help(cls, section): + section.set_title("SoS Cleaner Detailed Help") + section.add_text(cls.__doc__) + def load_map_file(self): """Verifies that the map file exists and has usable content. @@ -168,6 +220,8 @@ except KeyboardInterrupt: self.ui_log.info("\nExiting on user cancel") self._exit(130) + except Exception as e: + self._exit(1, e) @classmethod def add_parser_options(cls, parser): @@ -178,6 +232,11 @@ ) clean_grp.add_argument('target', metavar='TARGET', help='The directory or archive to obfuscate') + clean_grp.add_argument('--archive-type', default='auto', + choices=['auto', 'report', 'collect', + 'insights', 'data-dir', 'tarball'], + help=('Specify what kind of archive the target ' + 'was generated as')) clean_grp.add_argument('--domains', action='extend', default=[], help='List of domain names to obfuscate') clean_grp.add_argument('-j', '--jobs', default=4, type=int, @@ -218,59 +277,28 @@ In the event the target path is not an archive, abort. """ - if not tarfile.is_tarfile(self.opts.target): - self.ui_log.error( - "Invalid target: must be directory or tar archive" - ) - self._exit(1) - - archive = tarfile.open(self.opts.target) - self.arc_name = self.opts.target.split('/')[-1].split('.')[:-2][0] - - try: - archive.getmember(os.path.join(self.arc_name, 'sos_logs')) - except Exception: - # this is not an sos archive - self.ui_log.error("Invalid target: not an sos archive") - self._exit(1) - - # see if there are archives within this archive - nested_archives = [] - for _file in archive.getmembers(): - if (re.match('sosreport-.*.tar', _file.name.split('/')[-1]) and not - (_file.name.endswith(('.md5', '.sha256')))): - nested_archives.append(_file.name.split('/')[-1]) - - if nested_archives: - self.log_info("Found nested archive(s), extracting top level") - nested_path = self.extract_archive(archive) - for arc_file in os.listdir(nested_path): - if re.match('sosreport.*.tar.*', arc_file): - if arc_file.endswith(('.md5', '.sha256')): - continue - self.report_paths.append(os.path.join(nested_path, - arc_file)) - # add the toplevel extracted archive - self.report_paths.append(nested_path) + _arc = None + if self.opts.archive_type != 'auto': + check_type = self.opts.archive_type.replace('-', '_') + for archive in self.archive_types: + if archive.type_name == check_type: + _arc = archive(self.opts.target, self.tmpdir) else: - self.report_paths.append(self.opts.target) - - archive.close() - - def extract_archive(self, archive): - """Extract an archive into our tmpdir so that we may inspect it or - iterate through its contents for obfuscation - - Positional arguments: - - :param archive: An open TarFile object for the archive - - """ - if not isinstance(archive, tarfile.TarFile): - archive = tarfile.open(archive) - path = os.path.join(self.tmpdir, 'cleaner') - archive.extractall(path) - return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0]) + for arc in self.archive_types: + if arc.check_is_type(self.opts.target): + _arc = arc(self.opts.target, self.tmpdir) + break + if not _arc: + return + self.report_paths.append(_arc) + if _arc.is_nested: + self.report_paths.extend(_arc.get_nested_archives()) + # We need to preserve the top level archive until all + # nested archives are processed + self.report_paths.remove(_arc) + self.nested_archive = _arc + if self.nested_archive: + self.nested_archive.ui_name = self.nested_archive.description def execute(self): """SoSCleaner will begin by inspecting the TARGET option to determine @@ -283,6 +311,7 @@ be unpacked, cleaned, and repacked and the final top-level archive will then be repacked as well. """ + self.arc_name = self.opts.target.split('/')[-1].split('.tar')[0] if self.from_cmdline: self.print_disclaimer() self.report_paths = [] @@ -290,28 +319,17 @@ self.ui_log.error("Invalid target: no such file or directory %s" % self.opts.target) self._exit(1) - if os.path.isdir(self.opts.target): - self.arc_name = self.opts.target.split('/')[-1] - for _file in os.listdir(self.opts.target): - if _file == 'sos_logs': - self.report_paths.append(self.opts.target) - if (_file.startswith('sosreport') and - (_file.endswith(".tar.gz") or _file.endswith(".tar.xz"))): - self.report_paths.append(os.path.join(self.opts.target, - _file)) - if not self.report_paths: - self.ui_log.error("Invalid target: not an sos directory") - self._exit(1) - else: - self.inspect_target_archive() + + self.inspect_target_archive() if not self.report_paths: - self.ui_log.error("No valid sos archives or directories found\n") + self.ui_log.error("No valid archives or directories found\n") self._exit(1) # we have at least one valid target to obfuscate self.completed_reports = [] self.preload_all_archives_into_maps() + self.generate_parser_item_regexes() self.obfuscate_report_paths() if not self.completed_reports: @@ -334,33 +352,7 @@ final_path = None if len(self.completed_reports) > 1: - # we have an archive of archives, so repack the obfuscated tarball - arc_name = self.arc_name + '-obfuscated' - self.setup_archive(name=arc_name) - for arc in self.completed_reports: - if arc.is_tarfile: - arc_dest = self.obfuscate_string( - arc.final_archive_path.split('/')[-1] - ) - self.archive.add_file(arc.final_archive_path, - dest=arc_dest) - checksum = self.get_new_checksum(arc.final_archive_path) - if checksum is not None: - dname = self.obfuscate_string( - "checksums/%s.%s" % (arc_dest, self.hash_name) - ) - self.archive.add_string(checksum, dest=dname) - else: - for dirname, dirs, files in os.walk(arc.archive_path): - for filename in files: - if filename.startswith('sosreport'): - continue - fname = os.path.join(dirname, filename) - dnm = self.obfuscate_string( - fname.split(arc.archive_name)[-1].lstrip('/') - ) - self.archive.add_file(fname, dest=dnm) - arc_path = self.archive.finalize(self.opts.compression_type) + arc_path = self.rebuild_nested_archive() else: arc = self.completed_reports[0] arc_path = arc.final_archive_path @@ -371,8 +363,7 @@ ) with open(os.path.join(self.sys_tmp, chksum_name), 'w') as cf: cf.write(checksum) - - self.write_cleaner_log() + self.write_cleaner_log() final_path = self.obfuscate_string( os.path.join(self.sys_tmp, arc_path.split('/')[-1]) @@ -393,6 +384,30 @@ self.cleanup() + def rebuild_nested_archive(self): + """Handles repacking the nested tarball, now containing only obfuscated + copies of the reports, log files, manifest, etc... + """ + # we have an archive of archives, so repack the obfuscated tarball + arc_name = self.arc_name + '-obfuscated' + self.setup_archive(name=arc_name) + for archive in self.completed_reports: + arc_dest = archive.final_archive_path.split('/')[-1] + checksum = self.get_new_checksum(archive.final_archive_path) + if checksum is not None: + dname = "checksums/%s.%s" % (arc_dest, self.hash_name) + self.archive.add_string(checksum, dest=dname) + for dirn, dirs, files in os.walk(self.nested_archive.extracted_path): + for filename in files: + fname = os.path.join(dirn, filename) + dname = fname.split(self.nested_archive.extracted_path)[-1] + dname = dname.lstrip('/') + self.archive.add_file(fname, dest=dname) + # remove it now so we don't balloon our fs space needs + os.remove(fname) + self.write_cleaner_log(archive=True) + return self.archive.finalize(self.opts.compression_type) + def compile_mapping_dict(self): """Build a dict that contains each parser's map as a key, with the contents as that key's value. This will then be written to disk in the @@ -441,7 +456,7 @@ self.log_error("Could not update mapping config file: %s" % err) - def write_cleaner_log(self): + def write_cleaner_log(self, archive=False): """When invoked via the command line, the logging from SoSCleaner will not be added to the archive(s) it processes, so we need to write it separately to disk @@ -454,6 +469,10 @@ for line in self.sos_log_file.readlines(): logfile.write(line) + if archive: + self.obfuscate_file(log_name) + self.archive.add_file(log_name, dest="sos_logs/cleaner.log") + def get_new_checksum(self, archive_path): """Calculate a new checksum for the obfuscated archive, as the previous checksum will no longer be valid @@ -481,11 +500,11 @@ be obfuscated concurrently. """ try: - if len(self.report_paths) > 1: - msg = ("Found %s total reports to obfuscate, processing up to " - "%s concurrently\n" - % (len(self.report_paths), self.opts.jobs)) - self.ui_log.info(msg) + msg = ( + "Found %s total reports to obfuscate, processing up to %s " + "concurrently\n" % (len(self.report_paths), self.opts.jobs) + ) + self.ui_log.info(msg) if self.opts.keep_binary_files: self.ui_log.warning( "WARNING: binary files that potentially contain sensitive " @@ -494,53 +513,80 @@ pool = ThreadPoolExecutor(self.opts.jobs) pool.map(self.obfuscate_report, self.report_paths, chunksize=1) pool.shutdown(wait=True) + # finally, obfuscate the nested archive if one exists + if self.nested_archive: + self._replace_obfuscated_archives() + self.obfuscate_report(self.nested_archive) except KeyboardInterrupt: self.ui_log.info("Exiting on user cancel") os._exit(130) + def _replace_obfuscated_archives(self): + """When we have a nested archive, we need to rebuild the original + archive, which entails replacing the existing archives with their + obfuscated counterparts + """ + for archive in self.completed_reports: + os.remove(archive.archive_path) + dest = self.nested_archive.extracted_path + tarball = archive.final_archive_path.split('/')[-1] + dest_name = os.path.join(dest, tarball) + shutil.move(archive.final_archive_path, dest) + archive.final_archive_path = dest_name + + def generate_parser_item_regexes(self): + """For the parsers that use prebuilt lists of items, generate those + regexes now since all the parsers should be preloaded by the archive(s) + as well as being handed cmdline options and mapping file configuration. + """ + for parser in self.parsers: + parser.generate_item_regexes() + def preload_all_archives_into_maps(self): """Before doing the actual obfuscation, if we have multiple archives to obfuscate then we need to preload each of them into the mappings to ensure that node1 is obfuscated in node2 as well as node2 being obfuscated in node1's archive. """ - self.log_info("Pre-loading multiple archives into obfuscation maps") + self.log_info("Pre-loading all archives into obfuscation maps") for _arc in self.report_paths: - is_dir = os.path.isdir(_arc) - if is_dir: - _arc_name = _arc - else: - archive = tarfile.open(_arc) - _arc_name = _arc.split('/')[-1].split('.tar')[0] - # for each parser, load the map_prep_file into memory, and then - # send that for obfuscation. We don't actually obfuscate the file - # here, do that in the normal archive loop for _parser in self.parsers: - if not _parser.prep_map_file: + try: + pfile = _arc.prep_files[_parser.name.lower().split()[0]] + if not pfile: + continue + except (IndexError, KeyError): continue - if isinstance(_parser.prep_map_file, str): - _parser.prep_map_file = [_parser.prep_map_file] - for parse_file in _parser.prep_map_file: - _arc_path = os.path.join(_arc_name, parse_file) + if isinstance(pfile, str): + pfile = [pfile] + for parse_file in pfile: + self.log_debug("Attempting to load %s" % parse_file) try: - if is_dir: - _pfile = open(_arc_path, 'r') - content = _pfile.read() - else: - _pfile = archive.extractfile(_arc_path) - content = _pfile.read().decode('utf-8') - _pfile.close() + content = _arc.get_file_content(parse_file) + if not content: + continue if isinstance(_parser, SoSUsernameParser): _parser.load_usernames_into_map(content) - for line in content.splitlines(): - if isinstance(_parser, SoSHostnameParser): - _parser.load_hostname_into_map(line) - self.obfuscate_line(line) + elif isinstance(_parser, SoSHostnameParser): + if 'hostname' in parse_file: + _parser.load_hostname_into_map( + content.splitlines()[0] + ) + elif 'etc/hosts' in parse_file: + _parser.load_hostname_from_etc_hosts( + content + ) + else: + for line in content.splitlines(): + self.obfuscate_line(line) except Exception as err: - self.log_debug("Could not prep %s: %s" - % (_arc_path, err)) + self.log_info( + "Could not prepare %s from %s (archive: %s): %s" + % (_parser.name, parse_file, _arc.archive_name, + err) + ) - def obfuscate_report(self, report): + def obfuscate_report(self, archive): """Individually handle each archive or directory we've discovered by running through each file therein. @@ -549,17 +595,12 @@ :param report str: Filepath to the directory or archive """ try: - if not os.access(report, os.W_OK): - msg = "Insufficient permissions on %s" % report - self.log_info(msg) - self.ui_log.error(msg) - return - - archive = SoSObfuscationArchive(report, self.tmpdir) arc_md = self.cleaner_md.add_section(archive.archive_name) start_time = datetime.now() arc_md.add_field('start_time', start_time) - archive.extract() + # don't double extract nested archives + if not archive.is_extracted: + archive.extract() archive.report_msg("Beginning obfuscation...") file_list = archive.get_file_list() @@ -586,27 +627,28 @@ caller=archive.archive_name) # if the archive was already a tarball, repack it - method = archive.get_compression() - if method: - archive.report_msg("Re-compressing...") - try: - archive.rename_top_dir( - self.obfuscate_string(archive.archive_name) - ) - archive.compress(method) - except Exception as err: - self.log_debug("Archive %s failed to compress: %s" - % (archive.archive_name, err)) - archive.report_msg("Failed to re-compress archive: %s" - % err) - return + if not archive.is_nested: + method = archive.get_compression() + if method: + archive.report_msg("Re-compressing...") + try: + archive.rename_top_dir( + self.obfuscate_string(archive.archive_name) + ) + archive.compress(method) + except Exception as err: + self.log_debug("Archive %s failed to compress: %s" + % (archive.archive_name, err)) + archive.report_msg("Failed to re-compress archive: %s" + % err) + return + self.completed_reports.append(archive) end_time = datetime.now() arc_md.add_field('end_time', end_time) arc_md.add_field('run_time', end_time - start_time) arc_md.add_field('files_obfuscated', len(archive.file_sub_list)) arc_md.add_field('total_substitutions', archive.total_sub_count) - self.completed_reports.append(archive) rmsg = '' if archive.removed_file_count: rmsg = " [removed %s unprocessable files]" @@ -615,7 +657,7 @@ except Exception as err: self.ui_log.info("Exception while processing %s: %s" - % (report, err)) + % (archive.archive_name, err)) def obfuscate_file(self, filename, short_name=None, arc_name=None): """Obfuscate and individual file, line by line. @@ -635,16 +677,24 @@ # the requested file doesn't exist in the archive return subs = 0 + if not short_name: + short_name = filename.split('/')[-1] if not os.path.islink(filename): # don't run the obfuscation on the link, but on the actual file # at some other point. self.log_debug("Obfuscating %s" % short_name or filename, caller=arc_name) tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) + _parsers = [ + _p for _p in self.parsers if not + any([ + re.match(p, short_name) for p in _p.skip_files + ]) + ] with open(filename, 'r') as fname: for line in fname: try: - line, count = self.obfuscate_line(line) + line, count = self.obfuscate_line(line, _parsers) subs += count tfile.write(line) except Exception as err: @@ -714,7 +764,7 @@ pass return string_data - def obfuscate_line(self, line): + def obfuscate_line(self, line, parsers=None): """Run a line through each of the obfuscation parsers, keeping a cumulative total of substitutions done on that particular line. @@ -722,6 +772,8 @@ :param line str: The raw line as read from the file being processed + :param parsers: A list of parser objects to obfuscate + with. If None, use all. Returns the fully obfuscated line and the number of substitutions made """ @@ -730,7 +782,9 @@ count = 0 if not line.strip(): return line, count - for parser in self.parsers: + if parsers is None: + parsers = self.parsers + for parser in parsers: try: line, _count = parser.parse_line(line) count += _count @@ -745,3 +799,5 @@ for parser in self.parsers: _sec = parse_sec.add_section(parser.name.replace(' ', '_').lower()) _sec.add_field('entries', len(parser.mapping.dataset.keys())) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/cleaner/mappings/hostname_map.py sosreport-4.3/sos/cleaner/mappings/hostname_map.py --- sosreport-4.2/sos/cleaner/mappings/hostname_map.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/mappings/hostname_map.py 2022-02-15 04:20:20.000000000 +0000 @@ -40,6 +40,9 @@ 'api' ] + strip_exts = ('.yaml', '.yml', '.crt', '.key', '.pem', '.log', '.repo', + '.rules') + host_count = 0 domain_count = 0 _domains = {} @@ -50,10 +53,14 @@ in this parser, we need to re-inject entries from the map_file into these dicts and not just the underlying 'dataset' dict """ - for domain in self.dataset: + for domain, ob_pair in self.dataset.items(): if len(domain.split('.')) == 1: self.hosts[domain.split('.')[0]] = self.dataset[domain] else: + if ob_pair.startswith('obfuscateddomain'): + # directly exact domain matches + self._domains[domain] = ob_pair.split('.')[0] + continue # strip the host name and trailing top-level domain so that # we in inject the domain properly for later string matching @@ -101,15 +108,16 @@ """Check if a potential domain is in one of the domains we've loaded and should be obfuscated """ + if domain in self._domains: + return True host = domain.split('.') + no_tld = '.'.join(domain.split('.')[0:-1]) if len(host) == 1: # don't block on host's shortname - return host[0] in self.hosts.keys() - else: - domain = host[0:-1] - for known_domain in self._domains: - if known_domain in domain: - return True + return host[0] in self.hosts + elif any([no_tld.endswith(_d) for _d in self._domains]): + return True + return False def get(self, item): @@ -129,7 +137,7 @@ item = item[0:-1] if not self.domain_name_in_loaded_domains(item.lower()): return item - if item.endswith(('.yaml', '.yml', '.crt', '.key', '.pem')): + if item.endswith(self.strip_exts): ext = '.' + item.split('.')[-1] item = item.replace(ext, '') suffix += ext @@ -148,7 +156,8 @@ if len(_test) == 1 or not _test[0]: # does not match existing obfuscation continue - elif _test[0].endswith('.') and not _host_substr: + elif not _host_substr and (_test[0].endswith('.') or + item.endswith(_existing)): # new hostname in known domain final = super(SoSHostnameMap, self).get(item) break @@ -169,25 +178,34 @@ def sanitize_item(self, item): host = item.split('.') - if all([h.isupper() for h in host]): - # by convention we have just a domain - _host = [h.lower() for h in host] - return self.sanitize_domain(_host).upper() if len(host) == 1: # we have a shortname for a host - return self.sanitize_short_name(host[0]) + return self.sanitize_short_name(host[0].lower()) if len(host) == 2: # we have just a domain name, e.g. example.com - return self.sanitize_domain(host) + dname = self.sanitize_domain(host) + if all([h.isupper() for h in host]): + dname = dname.upper() + return dname if len(host) > 2: # we have an FQDN, e.g. foo.example.com hostname = host[0] domain = host[1:] # obfuscate the short name - ob_hostname = self.sanitize_short_name(hostname) + if len(hostname) > 2: + ob_hostname = self.sanitize_short_name(hostname.lower()) + else: + # by best practice it appears the host part of the fqdn was cut + # off due to some form of truncating, as such don't obfuscate + # short strings that are likely to throw off obfuscation of + # unrelated bits and paths + ob_hostname = 'unknown' ob_domain = self.sanitize_domain(domain) self.dataset[item] = ob_domain - return '.'.join([ob_hostname, ob_domain]) + _fqdn = '.'.join([ob_hostname, ob_domain]) + if all([h.isupper() for h in host]): + _fqdn = _fqdn.upper() + return _fqdn def sanitize_short_name(self, hostname): """Obfuscate the short name of the host with an incremented counter @@ -210,8 +228,8 @@ # don't obfuscate vendor domains if re.match(_skip, '.'.join(domain)): return '.'.join(domain) - top_domain = domain[-1] - dname = '.'.join(domain[0:-1]) + top_domain = domain[-1].lower() + dname = '.'.join(domain[0:-1]).lower() ob_domain = self._new_obfuscated_domain(dname) ob_domain = '.'.join([ob_domain, top_domain]) self.dataset['.'.join(domain)] = ob_domain diff -Nru sosreport-4.2/sos/cleaner/mappings/__init__.py sosreport-4.3/sos/cleaner/mappings/__init__.py --- sosreport-4.2/sos/cleaner/mappings/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/mappings/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -49,6 +49,8 @@ :param item: The plaintext object to obfuscate """ with self.lock: + if not item: + return item self.dataset[item] = self.sanitize_item(item) return self.dataset[item] @@ -67,7 +69,8 @@ """Retrieve an item's obfuscated counterpart from the map. If the item does not yet exist in the map, add it by generating one on the fly """ - if self.ignore_item(item) or self.item_in_dataset_values(item): + if (not item or self.ignore_item(item) or + self.item_in_dataset_values(item)): return item if item not in self.dataset: return self.add(item) diff -Nru sosreport-4.2/sos/cleaner/mappings/username_map.py sosreport-4.3/sos/cleaner/mappings/username_map.py --- sosreport-4.2/sos/cleaner/mappings/username_map.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/mappings/username_map.py 2022-02-15 04:20:20.000000000 +0000 @@ -24,7 +24,7 @@ def load_names_from_options(self, opt_names): for name in opt_names: - if name not in self.dataset.keys(): + if name and name not in self.dataset.keys(): self.add(name) def sanitize_item(self, username): @@ -33,5 +33,5 @@ ob_name = "obfuscateduser%s" % self.name_count self.name_count += 1 if ob_name in self.dataset.values(): - return self.sanitize_item(username) + return self.sanitize_item(username.lower()) return ob_name diff -Nru sosreport-4.2/sos/cleaner/obfuscation_archive.py sosreport-4.3/sos/cleaner/obfuscation_archive.py --- sosreport-4.2/sos/cleaner/obfuscation_archive.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/obfuscation_archive.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,319 +0,0 @@ -# Copyright 2020 Red Hat, Inc. Jake Hunsaker - -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -import logging -import os -import shutil -import stat -import tarfile -import re - -from concurrent.futures import ProcessPoolExecutor - - -# python older than 3.8 will hit a pickling error when we go to spawn a new -# process for extraction if this method is a part of the SoSObfuscationArchive -# class. So, the simplest solution is to remove it from the class. -def extract_archive(archive_path, tmpdir): - archive = tarfile.open(archive_path) - path = os.path.join(tmpdir, 'cleaner') - archive.extractall(path) - archive.close() - return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0]) - - -class SoSObfuscationArchive(): - """A representation of an extracted archive or an sos archive build - directory which is used by SoSCleaner. - - Each archive that needs to be obfuscated is loaded into an instance of this - class. All report-level operations should be contained within this class. - """ - - file_sub_list = [] - total_sub_count = 0 - removed_file_count = 0 - - def __init__(self, archive_path, tmpdir): - self.archive_path = archive_path - self.final_archive_path = self.archive_path - self.tmpdir = tmpdir - self.archive_name = self.archive_path.split('/')[-1].split('.tar')[0] - self.ui_name = self.archive_name - self.soslog = logging.getLogger('sos') - self.ui_log = logging.getLogger('sos_ui') - self.skip_list = self._load_skip_list() - self.log_info("Loaded %s as an archive" % self.archive_path) - - def report_msg(self, msg): - """Helper to easily format ui messages on a per-report basis""" - self.ui_log.info("{:<50} {}".format(self.ui_name + ' :', msg)) - - def _fmt_log_msg(self, msg): - return "[cleaner:%s] %s" % (self.archive_name, msg) - - def log_debug(self, msg): - self.soslog.debug(self._fmt_log_msg(msg)) - - def log_info(self, msg): - self.soslog.info(self._fmt_log_msg(msg)) - - def _load_skip_list(self): - """Provide a list of files and file regexes to skip obfuscation on - - Returns: list of files and file regexes - """ - return [ - 'sosreport-', - 'sys/firmware', - 'sys/fs', - 'sys/kernel/debug', - 'sys/module' - ] - - @property - def is_tarfile(self): - try: - return tarfile.is_tarfile(self.archive_path) - except Exception: - return False - - def remove_file(self, fname): - """Remove a file from the archive. This is used when cleaner encounters - a binary file, which we cannot reliably obfuscate. - """ - full_fname = self.get_file_path(fname) - # don't call a blank remove() here - if full_fname: - self.log_info("Removing binary file '%s' from archive" % fname) - os.remove(full_fname) - self.removed_file_count += 1 - - def extract(self): - if self.is_tarfile: - self.report_msg("Extracting...") - self.extracted_path = self.extract_self() - else: - self.extracted_path = self.archive_path - # if we're running as non-root (e.g. collector), then we can have a - # situation where a particular path has insufficient permissions for - # us to rewrite the contents and/or add it to the ending tarfile. - # Unfortunately our only choice here is to change the permissions - # that were preserved during report collection - if os.getuid() != 0: - self.log_debug('Verifying permissions of archive contents') - for dirname, dirs, files in os.walk(self.extracted_path): - try: - for _dir in dirs: - _dirname = os.path.join(dirname, _dir) - _dir_perms = os.stat(_dirname).st_mode - os.chmod(_dirname, _dir_perms | stat.S_IRWXU) - for filename in files: - fname = os.path.join(dirname, filename) - # protect against symlink race conditions - if not os.path.exists(fname) or os.path.islink(fname): - continue - if (not os.access(fname, os.R_OK) or not - os.access(fname, os.W_OK)): - self.log_debug( - "Adding owner rw permissions to %s" - % fname.split(self.archive_path)[-1] - ) - os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR) - except Exception as err: - self.log_debug("Error while trying to set perms: %s" % err) - self.log_debug("Extracted path is %s" % self.extracted_path) - - def rename_top_dir(self, new_name): - """Rename the top-level directory to new_name, which should be an - obfuscated string that scrubs the hostname from the top-level dir - which would be named after the unobfuscated sos report - """ - _path = self.extracted_path.replace(self.archive_name, new_name) - self.archive_name = new_name - os.rename(self.extracted_path, _path) - self.extracted_path = _path - - def get_compression(self): - """Return the compression type used by the archive, if any. This is - then used by SoSCleaner to generate a policy-derived compression - command to repack the archive - """ - if self.is_tarfile: - if self.archive_path.endswith('xz'): - return 'xz' - return 'gz' - return None - - def build_tar_file(self, method): - """Pack the extracted archive as a tarfile to then be re-compressed - """ - mode = 'w' - tarpath = self.extracted_path + '-obfuscated.tar' - compr_args = {} - if method: - mode += ":%s" % method - tarpath += ".%s" % method - if method == 'xz': - compr_args = {'preset': 3} - else: - compr_args = {'compresslevel': 6} - self.log_debug("Building tar file %s" % tarpath) - tar = tarfile.open(tarpath, mode=mode, **compr_args) - tar.add(self.extracted_path, - arcname=os.path.split(self.archive_name)[1]) - tar.close() - return tarpath - - def compress(self, method): - """Execute the compression command, and set the appropriate final - archive path for later reference by SoSCleaner on a per-archive basis - """ - try: - self.final_archive_path = self.build_tar_file(method) - except Exception as err: - self.log_debug("Exception while re-compressing archive: %s" % err) - raise - self.log_debug("Compressed to %s" % self.final_archive_path) - try: - self.remove_extracted_path() - except Exception as err: - self.log_debug("Failed to remove extraction directory: %s" % err) - self.report_msg('Failed to remove temporary extraction directory') - - def remove_extracted_path(self): - """After the tarball has been re-compressed, remove the extracted path - so that we don't take up that duplicate space any longer during - execution - """ - def force_delete_file(action, name, exc): - os.chmod(name, stat.S_IWUSR) - if os.path.isfile(name): - os.remove(name) - else: - shutil.rmtree(name) - self.log_debug("Removing %s" % self.extracted_path) - shutil.rmtree(self.extracted_path, onerror=force_delete_file) - - def extract_self(self): - """Extract an archive into our tmpdir so that we may inspect it or - iterate through its contents for obfuscation - """ - - with ProcessPoolExecutor(1) as _pool: - _path_future = _pool.submit(extract_archive, - self.archive_path, self.tmpdir) - path = _path_future.result() - return path - - def get_file_list(self): - """Return a list of all files within the archive""" - self.file_list = [] - for dirname, dirs, files in os.walk(self.extracted_path): - for _dir in dirs: - _dirpath = os.path.join(dirname, _dir) - # catch dir-level symlinks - if os.path.islink(_dirpath) and os.path.isdir(_dirpath): - self.file_list.append(_dirpath) - for filename in files: - self.file_list.append(os.path.join(dirname, filename)) - return self.file_list - - def get_directory_list(self): - """Return a list of all directories within the archive""" - dir_list = [] - for dirname, dirs, files in os.walk(self.extracted_path): - dir_list.append(dirname) - return dir_list - - def update_sub_count(self, fname, count): - """Called when a file has finished being parsed and used to track - total substitutions made and number of files that had changes made - """ - self.file_sub_list.append(fname) - self.total_sub_count += count - - def get_file_path(self, fname): - """Return the filepath of a specific file within the archive so that - it may be selectively inspected if it exists - """ - _path = os.path.join(self.extracted_path, fname.lstrip('/')) - return _path if os.path.exists(_path) else '' - - def should_skip_file(self, filename): - """Checks the provided filename against a list of filepaths to not - perform obfuscation on, as defined in self.skip_list - - Positional arguments: - - :param filename str: Filename relative to the extracted - archive root - """ - - if (not os.path.isfile(self.get_file_path(filename)) and not - os.path.islink(self.get_file_path(filename))): - return True - - for _skip in self.skip_list: - if filename.startswith(_skip) or re.match(_skip, filename): - return True - return False - - def should_remove_file(self, fname): - """Determine if the file should be removed or not, due to an inability - to reliably obfuscate that file based on the filename. - - :param fname: Filename relative to the extracted archive root - :type fname: ``str`` - - :returns: ``True`` if the file cannot be reliably obfuscated - :rtype: ``bool`` - """ - obvious_removes = [ - r'.*\.gz', # TODO: support flat gz/xz extraction - r'.*\.xz', - r'.*\.bzip2', - r'.*\.tar\..*', # TODO: support archive unpacking - r'.*\.txz$', - r'.*\.tgz$', - r'.*\.bin', - r'.*\.journal', - r'.*\~$' - ] - - # if the filename matches, it is obvious we can remove them without - # doing the read test - for _arc_reg in obvious_removes: - if re.match(_arc_reg, fname): - return True - - if os.path.isfile(self.get_file_path(fname)): - return self.file_is_binary(fname) - # don't fail on dir-level symlinks - return False - - def file_is_binary(self, fname): - """Determine if the file is a binary file or not. - - - :param fname: Filename relative to the extracted archive root - :type fname: ``str`` - - :returns: ``True`` if file is binary, else ``False`` - :rtype: ``bool`` - """ - with open(self.get_file_path(fname), 'tr') as tfile: - try: - # when opened as above (tr), reading binary content will raise - # an exception - tfile.read(1) - return False - except UnicodeDecodeError: - return True diff -Nru sosreport-4.2/sos/cleaner/parsers/hostname_parser.py sosreport-4.3/sos/cleaner/parsers/hostname_parser.py --- sosreport-4.2/sos/cleaner/parsers/hostname_parser.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/parsers/hostname_parser.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,6 +8,8 @@ # # See the LICENSE file in the source distribution for further information. +import re + from sos.cleaner.parsers import SoSCleanerParser from sos.cleaner.mappings.hostname_map import SoSHostnameMap @@ -16,7 +18,6 @@ name = 'Hostname Parser' map_file_key = 'hostname_map' - prep_map_file = 'sos_commands/host/hostname' regex_patterns = [ r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}(\b|_)))' ] @@ -62,6 +63,25 @@ self.mapping.add(high_domain) self.mapping.add(hostname_string) + def load_hostname_from_etc_hosts(self, content): + """Parse an archive's copy of /etc/hosts, which requires handling that + is separate from the output of the `hostname` command. Just like + load_hostname_into_map(), this has to be done explicitly and we + cannot rely upon the more generic methods to do this reliably. + """ + lines = content.splitlines() + for line in lines: + if line.startswith('#') or 'localhost' in line: + continue + hostln = line.split()[1:] + for host in hostln: + if len(host.split('.')) == 1: + # only generate a mapping for fqdns but still record the + # short name here for later obfuscation with parse_line() + self.short_names.append(host) + else: + self.mapping.add(host) + def parse_line(self, line): """Override the default parse_line() method to also check for the shortname of the host derived from the hostname. @@ -73,9 +93,9 @@ """ if search in self.mapping.skip_keys: return ln, count - if search in ln: - count += ln.count(search) - ln = ln.replace(search, self.mapping.get(repl or search)) + _reg = re.compile(search, re.I) + if _reg.search(ln): + return _reg.subn(self.mapping.get(repl or search), ln) return ln, count count = 0 diff -Nru sosreport-4.2/sos/cleaner/parsers/__init__.py sosreport-4.3/sos/cleaner/parsers/__init__.py --- sosreport-4.2/sos/cleaner/parsers/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/parsers/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -37,11 +37,6 @@ :cvar map_file_key: The key in the ``map_file`` to read when loading previous obfuscation matches :vartype map_file_key: ``str`` - - - :cvar prep_map_file: File to read from an archive to pre-seed the map with - matches. E.G. ip_addr for loading IP addresses - :vartype prep_map_fie: ``str`` """ name = 'Undefined Parser' @@ -49,12 +44,21 @@ skip_line_patterns = [] skip_files = [] map_file_key = 'unset' - prep_map_file = [] def __init__(self, config={}): + self.regexes = {} if self.map_file_key in config: self.mapping.conf_update(config[self.map_file_key]) + def generate_item_regexes(self): + """Generate regexes for items the parser will be searching for + repeatedly without needing to generate them for every file and/or line + we process + + Not used by all parsers. + """ + pass + def parse_line(self, line): """This will be called for every line in every file we process, so that every parser has a chance to scrub everything. diff -Nru sosreport-4.2/sos/cleaner/parsers/ip_parser.py sosreport-4.3/sos/cleaner/parsers/ip_parser.py --- sosreport-4.2/sos/cleaner/parsers/ip_parser.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/parsers/ip_parser.py 2022-02-15 04:20:20.000000000 +0000 @@ -37,11 +37,11 @@ 'sos_commands/snappy/snap_list_--all', 'sos_commands/snappy/snap_--version', 'sos_commands/vulkan/vulkaninfo', - 'var/log/.*dnf.*' + 'var/log/.*dnf.*', + 'var/log/.*packag.*' # get 'packages' and 'packaging' logs ] map_file_key = 'ip_map' - prep_map_file = 'sos_commands/networking/ip_-o_addr' def __init__(self, config): self.mapping = SoSIPMap() diff -Nru sosreport-4.2/sos/cleaner/parsers/keyword_parser.py sosreport-4.3/sos/cleaner/parsers/keyword_parser.py --- sosreport-4.2/sos/cleaner/parsers/keyword_parser.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/parsers/keyword_parser.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,6 +9,7 @@ # See the LICENSE file in the source distribution for further information. import os +import re from sos.cleaner.parsers import SoSCleanerParser from sos.cleaner.mappings.keyword_map import SoSKeywordMap @@ -20,7 +21,6 @@ name = 'Keyword Parser' map_file_key = 'keyword_map' - prep_map_file = '' def __init__(self, config, keywords=None, keyword_file=None): self.mapping = SoSKeywordMap() @@ -34,16 +34,20 @@ # pre-generate an obfuscation mapping for each keyword # this is necessary for cases where filenames are being # obfuscated before or instead of file content - self.mapping.get(keyword) + self.mapping.get(keyword.lower()) self.user_keywords.append(keyword) if keyword_file and os.path.exists(keyword_file): with open(keyword_file, 'r') as kwf: self.user_keywords.extend(kwf.read().splitlines()) + def generate_item_regexes(self): + for kw in self.user_keywords: + self.regexes[kw] = re.compile(kw, re.I) + def parse_line(self, line): count = 0 - for keyword in sorted(self.user_keywords, reverse=True): - if keyword in line: - line = line.replace(keyword, self.mapping.get(keyword)) - count += 1 + for kwrd, reg in sorted(self.regexes.items(), key=len, reverse=True): + if reg.search(line): + line, _count = reg.subn(self.mapping.get(kwrd.lower()), line) + count += _count return line, count diff -Nru sosreport-4.2/sos/cleaner/parsers/mac_parser.py sosreport-4.3/sos/cleaner/parsers/mac_parser.py --- sosreport-4.2/sos/cleaner/parsers/mac_parser.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/cleaner/parsers/mac_parser.py 2022-02-15 04:20:20.000000000 +0000 @@ -13,24 +13,34 @@ import re +# aa:bb:cc:fe:ff:dd:ee:ff +IPV6_REG_8HEX = (r'((?8}{:<40}{:<30}".format(' ', 'Name', 'Description'), + newline=False + ) + for cluster in clusters: + _sec = bold("collect.clusters.%s" % cluster[0]) + section.add_text( + "{:>8}{:<40}{:<30}".format(' ', _sec, cluster[1].cluster_name), + newline=False + ) + def _get_options(self): """Loads the options defined by a cluster and sets the default value""" for opt in self.option_list: @@ -135,7 +237,7 @@ key rather than prompting the user for one or a password. Note this will only function if collector is being run locally on the - master node. + primary node. """ self.cluster_ssh_key = key @@ -149,31 +251,36 @@ """ pass - def set_master_options(self, node): + def set_transport_type(self): + """The default connection type used by sos collect is to leverage the + local system's SSH installation using ControlPersist, however certain + cluster types may want to use something else. + + Override this in a specific cluster profile to set the ``transport`` + option according to what type of transport should be used. + """ + return 'control_persist' + + def set_primary_options(self, node): """If there is a need to set specific options in the sos command being - run on the cluster's master nodes, override this method in the cluster + run on the cluster's primary nodes, override this method in the cluster profile and do that here. - :param node: The master node + :param node: The primary node :type node: ``SoSNode`` """ pass - def check_node_is_master(self, node): - """In the event there are multiple masters, or if the collect command + def check_node_is_primary(self, node): + """In the event there are multiple primaries, or if the collect command is being run from a system that is technically capable of enumerating - nodes but the cluster profiles needs to specify master-specific options - for other nodes, override this method in the cluster profile + nodes but the cluster profiles needs to specify primary-specific + options for other nodes, override this method in the cluster profile :param node: The node for the cluster to check :type node: ``SoSNode`` """ - return node.address == self.master.address - - def exec_master_cmd(self, cmd, need_root=False): - self.log_debug("Use of exec_master_cmd() is deprecated and will be " - "removed. Use exec_primary_cmd() instead") - return self.exec_primary_cmd(cmd, need_root) + return node.address == self.primary.address def exec_primary_cmd(self, cmd, need_root=False): """Used to retrieve command output from a (primary) node in a cluster @@ -187,9 +294,10 @@ :returns: The output and status of `cmd` :rtype: ``dict`` """ - res = self.master.run_command(cmd, get_pty=True, need_root=need_root) - if res['stdout']: - res['stdout'] = res['stdout'].replace('Password:', '') + pty = self.primary.local is False + res = self.primary.run_command(cmd, get_pty=pty, need_root=need_root) + if res['output']: + res['output'] = res['output'].replace('Password:', '') return res def setup(self): @@ -214,10 +322,20 @@ :rtype: ``bool`` """ for pkg in self.packages: - if self.master.is_installed(pkg): + if self.primary.is_installed(pkg): return True return False + def cleanup(self): + """ + This may be overridden by clusters + + Perform any necessary cleanup steps required by the cluster profile. + This helps ensure that sos does make lasting changes to the environment + in which we are running + """ + pass + def get_nodes(self): """ This MUST be overridden by a cluster profile subclassing this class @@ -255,8 +373,8 @@ def set_node_label(self, node): """This may be overridden by clusters profiles subclassing this class - If there is a distinction between masters and nodes, or types of nodes, - then this can be used to label the sosreport archive differently. + If there is a distinction between primaries and nodes, or types of + nodes, then this can be used to label the sosreport archive differently """ return '' diff -Nru sosreport-4.2/sos/collector/clusters/jbon.py sosreport-4.3/sos/collector/clusters/jbon.py --- sosreport-4.2/sos/collector/clusters/jbon.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/clusters/jbon.py 2022-02-15 04:20:20.000000000 +0000 @@ -12,11 +12,14 @@ class jbon(Cluster): - '''Just a Bunch of Nodes - - Used when --cluster-type=none (or jbon), to avoid cluster checks, and just - use the provided --nodes list - ''' + """ + Used when --cluster-type=none (or jbon) to avoid cluster checks, and just + use the provided --nodes list. + + Using this profile will skip any and all operations that a cluster profile + normally performs, and will not set any plugins, plugin options, or presets + for the sos report generated on the nodes provided by --nodes. + """ cluster_name = 'Just a Bunch Of Nodes (no cluster)' packages = None @@ -28,3 +31,5 @@ # This should never be called, but as insurance explicitly never # allow this to be enabled via the determine_cluster() path return False + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/clusters/kubernetes.py sosreport-4.3/sos/collector/clusters/kubernetes.py --- sosreport-4.2/sos/collector/clusters/kubernetes.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/clusters/kubernetes.py 2022-02-15 04:20:20.000000000 +0000 @@ -13,7 +13,12 @@ class kubernetes(Cluster): - + """ + The kuberentes cluster profile is intended to be used on kubernetes + clusters built from the upstream/source kubernetes (k8s) project. It is + not intended for use with other projects or platforms that are built ontop + of kubernetes. + """ cluster_name = 'Community Kubernetes' packages = ('kubernetes-master',) sos_plugins = ['kubernetes'] @@ -34,7 +39,7 @@ if res['status'] == 0: nodes = [] roles = [x for x in self.get_option('role').split(',') if x] - for nodeln in res['stdout'].splitlines()[1:]: + for nodeln in res['output'].splitlines()[1:]: node = nodeln.split() if not roles: nodes.append(node[0]) @@ -44,3 +49,5 @@ return nodes else: raise Exception('Node enumeration did not return usable output') + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/clusters/ocp.py sosreport-4.3/sos/collector/clusters/ocp.py --- sosreport-4.2/sos/collector/clusters/ocp.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/clusters/ocp.py 2022-02-15 04:20:20.000000000 +0000 @@ -12,20 +12,56 @@ from pipes import quote from sos.collector.clusters import Cluster +from sos.utilities import is_executable class ocp(Cluster): - """OpenShift Container Platform v4""" + """ + This profile is for use with OpenShift Container Platform (v4) clusters + instead of the kubernetes profile. + + This profile will favor using the `oc` transport type, which means it will + leverage a locally installed `oc` binary. This is also how node enumeration + is done. To instead use SSH to connect to the nodes, use the + '--transport=control_persist' option. + + Thus, a functional `oc` binary for the user executing sos collect is + required. Functional meaning that the user can run `oc` commands with + clusterAdmin privileges. + + If this requires the use of a secondary configuration file, specify that + path with the 'kubeconfig' cluster option. + + Alternatively, provide a clusterAdmin access token either via the 'token' + cluster option or, preferably, the SOSOCPTOKEN environment variable. + + By default, this profile will enumerate only master nodes within the + cluster, and this may be changed by overriding the 'role' cluster option. + To collect from all nodes in the cluster regardless of role, use the form + -c ocp.role=''. + + Filtering nodes by a label applied to that node is also possible via the + label cluster option, though be aware that this is _combined_ with the role + option mentioned above. + + To avoid redundant collections of OCP API information (e.g. 'oc get' + commands), this profile will attempt to enable the openshift plugin on only + a single master node. If the none of the master nodes have a functional + 'oc' binary available, *and* the --no-local option is used, that means that + no API data will be collected. + """ cluster_name = 'OpenShift Container Platform v4' packages = ('openshift-hyperkube', 'openshift-clients') api_collect_enabled = False token = None + project = 'sos-collect-tmp' + oc_cluster_admin = None option_list = [ ('label', '', 'Colon delimited list of labels to select nodes with'), - ('role', '', 'Colon delimited list of roles to select nodes with'), + ('role', 'master', 'Colon delimited list of roles to filter on'), ('kubeconfig', '', 'Path to the kubeconfig file'), ('token', '', 'Service account token to use for oc authorization') ] @@ -53,11 +89,48 @@ if self.token: self._attempt_oc_login() _who = self.fmt_oc_cmd('whoami') - return self.exec_master_cmd(_who)['status'] == 0 + return self.exec_primary_cmd(_who)['status'] == 0 + + def setup(self): + """Create the project that we will be executing in for any nodes' + collection via a container image + """ + if not self.set_transport_type() == 'oc': + return + + out = self.exec_primary_cmd(self.fmt_oc_cmd("auth can-i '*' '*'")) + self.oc_cluster_admin = out['status'] == 0 + if not self.oc_cluster_admin: + self.log_debug("Check for cluster-admin privileges returned false," + " cannot create project in OCP cluster") + raise Exception("Insufficient permissions to create temporary " + "collection project.\nAborting...") + + self.log_info("Creating new temporary project '%s'" % self.project) + ret = self.exec_primary_cmd("oc new-project %s" % self.project) + if ret['status'] == 0: + return True + + self.log_debug("Failed to create project: %s" % ret['output']) + raise Exception("Failed to create temporary project for collection. " + "\nAborting...") + + def cleanup(self): + """Remove the project we created to execute within + """ + if self.project: + ret = self.exec_primary_cmd("oc delete project %s" % self.project) + if not ret['status'] == 0: + self.log_error("Error deleting temporary project: %s" + % ret['output']) + # don't leave the config on a non-existing project + self.exec_primary_cmd("oc project default") + self.project = None + return True def _build_dict(self, nodelist): """From the output of get_nodes(), construct an easier-to-reference - dict of nodes that will be used in determining labels, master status, + dict of nodes that will be used in determining labels, primary status, etc... :param nodelist: The split output of `oc get nodes` @@ -83,6 +156,21 @@ nodes[_node[0]][column] = _node[idx[column]] return nodes + def set_transport_type(self): + if self.opts.transport != 'auto': + return self.opts.transport + if is_executable('oc'): + return 'oc' + self.log_info("Local installation of 'oc' not found or is not " + "correctly configured. Will use ControlPersist.") + self.ui_log.warn( + "Preferred transport 'oc' not available, will fallback to SSH." + ) + if not self.opts.batch: + input("Press ENTER to continue connecting with SSH, or Ctrl+C to" + "abort.") + return 'control_persist' + def get_nodes(self): nodes = [] self.node_dict = {} @@ -90,21 +178,25 @@ if self.get_option('label'): labels = ','.join(self.get_option('label').split(':')) cmd += " -l %s" % quote(labels) - res = self.exec_master_cmd(self.fmt_oc_cmd(cmd)) + res = self.exec_primary_cmd(self.fmt_oc_cmd(cmd)) if res['status'] == 0: + if self.get_option('role') == 'master': + self.log_warn("NOTE: By default, only master nodes are listed." + "\nTo collect from all/more nodes, override the " + "role option with '-c ocp.role=role1:role2'") roles = [r for r in self.get_option('role').split(':')] - self.node_dict = self._build_dict(res['stdout'].splitlines()) - for node in self.node_dict: + self.node_dict = self._build_dict(res['output'].splitlines()) + for node_name, node in self.node_dict.items(): if roles: for role in roles: - if role in node: - nodes.append(node) + if role == node['roles']: + nodes.append(node_name) else: - nodes.append(node) + nodes.append(node_name) else: msg = "'oc' command failed" - if 'Missing or incomplete' in res['stdout']: - msg = ("'oc' failed due to missing kubeconfig on master node." + if 'Missing or incomplete' in res['output']: + msg = ("'oc' failed due to missing kubeconfig on primary node." " Specify one via '-c ocp.kubeconfig='") raise Exception(msg) return nodes @@ -117,17 +209,17 @@ return label return '' - def check_node_is_master(self, sosnode): + def check_node_is_primary(self, sosnode): if sosnode.address not in self.node_dict: return False return 'master' in self.node_dict[sosnode.address]['roles'] - def set_master_options(self, node): + def set_primary_options(self, node): node.enable_plugins.append('openshift') if self.api_collect_enabled: # a primary has already been enabled for API collection, disable # it among others - node.plugin_options.append('openshift.no-oc=on') + node.plugopts.append('openshift.no-oc=on') else: _oc_cmd = 'oc' if node.host.containerized: @@ -167,4 +259,6 @@ def set_node_options(self, node): # don't attempt OC API collections on non-primary nodes - node.plugin_options.append('openshift.no-oc=on') + node.plugopts.append('openshift.no-oc=on') + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/clusters/ovirt.py sosreport-4.3/sos/collector/clusters/ovirt.py --- sosreport-4.2/sos/collector/clusters/ovirt.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/clusters/ovirt.py 2022-02-15 04:20:20.000000000 +0000 @@ -17,6 +17,33 @@ class ovirt(Cluster): + """ + This cluster profile is for the oVirt/RHV project which provides for a + virtualization cluster built ontop of KVM. + + Nodes enumerated will be hypervisors within the envrionment, not virtual + machines running on those hypervisors. By default, ALL hypervisors within + the environment are returned. This may be influenced by the 'cluster' and + 'datacenter' cluster options, which will limit enumeration to hypervisors + within the specific cluster and/or datacenter. The spm-only cluster option + may also be used to only collect from hypervisors currently holding the + SPM role. + + Optionally, to only collect an archive from manager and the postgresql + database, use the no-hypervisors cluster option. + + By default, a second archive from the manager will be collected that is + just the postgresql plugin configured in such a way that a dump of the + manager's database that can be explored and restored to other systems will + be collected. + + The ovirt profile focuses on the upstream, community ovirt project. + + The rhv profile is for Red Hat customers running RHV (formerly RHEV). + + The rhhi_virt profile is for Red Hat customers running RHV in a + hyper-converged setup and enables gluster collections. + """ cluster_name = 'Community oVirt' packages = ('ovirt-engine',) @@ -32,7 +59,7 @@ def _run_db_query(self, query): ''' - Wrapper for running DB queries on the master. Any scrubbing of the + Wrapper for running DB queries on the manager. Any scrubbing of the query should be done _before_ passing the query to this method. ''' cmd = "%s %s" % (self.db_exec, quote(query)) @@ -62,10 +89,10 @@ This only runs if we're locally on the RHV-M, *and* if no ssh-keys are called out on the command line, *and* no --password option is given. ''' - if self.master.local: + if self.primary.local: if not any([self.opts.ssh_key, self.opts.password, self.opts.password_per_node]): - if self.master.file_exists(ENGINE_KEY): + if self.primary.file_exists(ENGINE_KEY): self.add_default_ssh_key(ENGINE_KEY) self.log_debug("Found engine SSH key. User command line" " does not specify a key or password, using" @@ -98,7 +125,7 @@ return [] res = self._run_db_query(self.dbquery) if res['status'] == 0: - nodes = res['stdout'].splitlines()[2:-1] + nodes = res['output'].splitlines()[2:-1] return [n.split('(')[0].strip() for n in nodes] else: raise Exception('database query failed, return code: %s' @@ -114,7 +141,7 @@ engconf = '/etc/ovirt-engine/engine.conf.d/10-setup-database.conf' res = self.exec_primary_cmd('cat %s' % engconf, need_root=True) if res['status'] == 0: - config = res['stdout'].splitlines() + config = res['output'].splitlines() for line in config: try: k = str(line.split('=')[0]) @@ -141,11 +168,11 @@ '--batch -o postgresql {}' ).format(self.conf['ENGINE_DB_PASSWORD'], sos_opt) db_sos = self.exec_primary_cmd(cmd, need_root=True) - for line in db_sos['stdout'].splitlines(): + for line in db_sos['output'].splitlines(): if fnmatch.fnmatch(line, '*sosreport-*tar*'): _pg_dump = line.strip() - self.master.manifest.add_field('postgresql_dump', - _pg_dump.split('/')[-1]) + self.primary.manifest.add_field('postgresql_dump', + _pg_dump.split('/')[-1]) return _pg_dump self.log_error('Failed to gather database dump') return False @@ -158,7 +185,7 @@ sos_preset = 'rhv' def set_node_label(self, node): - if node.address == self.master.address: + if node.address == self.primary.address: return 'manager' if node.is_installed('ovirt-node-ng-nodectl'): return 'rhvh' @@ -174,11 +201,13 @@ sos_preset = 'rhv' def check_enabled(self): - return (self.master.is_installed('rhvm') and self._check_for_rhhiv()) + return (self.primary.is_installed('rhvm') and self._check_for_rhhiv()) def _check_for_rhhiv(self): ret = self._run_db_query('SELECT count(server_id) FROM gluster_server') if ret['status'] == 0: # if there are any entries in this table, RHHI-V is in use - return ret['stdout'].splitlines()[2].strip() != '0' + return ret['output'].splitlines()[2].strip() != '0' return False + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/clusters/pacemaker.py sosreport-4.3/sos/collector/clusters/pacemaker.py --- sosreport-4.2/sos/collector/clusters/pacemaker.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/clusters/pacemaker.py 2022-02-15 04:20:20.000000000 +0000 @@ -27,7 +27,7 @@ self.log_error('Cluster status could not be determined. Is the ' 'cluster running on this node?') return [] - if 'node names do not match' in self.res['stdout']: + if 'node names do not match' in self.res['output']: self.log_warn('Warning: node name mismatch reported. Attempts to ' 'connect to some nodes may fail.\n') return self.parse_pcs_output() @@ -41,17 +41,19 @@ return nodes def get_online_nodes(self): - for line in self.res['stdout'].splitlines(): + for line in self.res['output'].splitlines(): if line.startswith('Online:'): nodes = line.split('[')[1].split(']')[0] return [n for n in nodes.split(' ') if n] def get_offline_nodes(self): offline = [] - for line in self.res['stdout'].splitlines(): + for line in self.res['output'].splitlines(): if line.startswith('Node') and line.endswith('(offline)'): offline.append(line.split()[1].replace(':', '')) if line.startswith('OFFLINE:'): nodes = line.split('[')[1].split(']')[0] offline.extend([n for n in nodes.split(' ') if n]) return offline + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/clusters/satellite.py sosreport-4.3/sos/collector/clusters/satellite.py --- sosreport-4.2/sos/collector/clusters/satellite.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/clusters/satellite.py 2022-02-15 04:20:20.000000000 +0000 @@ -13,7 +13,14 @@ class satellite(Cluster): - """Red Hat Satellite 6""" + """ + This profile is specifically for Red Hat Satellite 6, and not earlier + releases of Satellite. + + While note technically a 'cluster' in the traditional sense, Satellite + does provide for 'capsule' nodes which is what this profile aims to + enumerate beyond the 'primary' Satellite system. + """ cluster_name = 'Red Hat Satellite 6' packages = ('satellite', 'satellite-installer') @@ -28,13 +35,15 @@ res = self.exec_primary_cmd(cmd, need_root=True) if res['status'] == 0: nodes = [ - n.strip() for n in res['stdout'].splitlines() + n.strip() for n in res['output'].splitlines() if 'could not change directory' not in n ] return nodes return [] def set_node_label(self, node): - if node.address == self.master.address: + if node.address == self.primary.address: return 'satellite' return 'capsule' + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/exceptions.py sosreport-4.3/sos/collector/exceptions.py --- sosreport-4.2/sos/collector/exceptions.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/exceptions.py 2022-02-15 04:20:20.000000000 +0000 @@ -94,6 +94,16 @@ super(UnsupportedHostException, self).__init__(message) +class InvalidTransportException(Exception): + """Raised when a transport is requested but it does not exist or is + not supported locally""" + + def __init__(self, transport=None): + message = ("Connection failed: unknown or unsupported transport %s" + % transport if transport else '') + super(InvalidTransportException, self).__init__(message) + + __all__ = [ 'AuthPermissionDeniedException', 'CommandTimeoutException', @@ -104,5 +114,6 @@ 'InvalidPasswordException', 'PasswordRequestException', 'TimeoutPasswordAuthException', - 'UnsupportedHostException' + 'UnsupportedHostException', + 'InvalidTransportException' ] diff -Nru sosreport-4.2/sos/collector/__init__.py sosreport-4.3/sos/collector/__init__.py --- sosreport-4.2/sos/collector/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -17,7 +17,6 @@ import string import socket import shutil -import subprocess import sys from datetime import datetime @@ -28,21 +27,43 @@ from textwrap import fill from sos.cleaner import SoSCleaner from sos.collector.sosnode import SosNode -from sos.collector.exceptions import ControlPersistUnsupportedException from sos.options import ClusterOption from sos.component import SoSComponent +from sos.utilities import bold from sos import __version__ COLLECTOR_CONFIG_DIR = '/etc/sos/groups.d' class SoSCollector(SoSComponent): - """Collector is the formerly standalone sos-collector project, brought into - sos natively in 4.0 + """ + sos collect, or SoS Collector, is the formerly standalone sos-collector + project, brought into sos natively in 4.0 and later. + + It is meant to collect sos reports from an arbitrary number of remote + nodes, as well as the localhost, at the same time. These nodes may be + either user defined, defined by some clustering software, or both. + + For cluster defined lists of nodes, cluster profiles exist that not only + define how these node lists are generated but may also influence the + sos report command run on nodes depending upon their role within the + cluster. + + Nodes are connected to via a 'transport' which defaults to the use of + OpenSSH's Control Persist feature. Other transport types are available, and + may be specifically linked to use with a certain cluster profile (or, at + minimum, a node within a certain cluster type even if that profile is not + used). + + sos collect may be run from either a node within the cluster that is + capable of enumerating/discovering the other cluster nodes, or may be run + from a user's workstation and instructed to first connect to such a node + via the --primary option. If run in the latter manner, users will likely + want to use the --no-local option, as by default sos collect will also + collect an sos report locally. - It is meant to collect reports from an arbitrary number of remote nodes, - as well as the localhost, at the same time. These nodes may be either user - defined, defined by some clustering software, or both. + Users should expect this command to result in a tarball containing one or + more sos report archives on the system that sos collect was executed on. """ desc = 'Collect an sos report from multiple nodes simultaneously' @@ -57,6 +78,7 @@ 'clean': False, 'cluster_options': [], 'cluster_type': None, + 'container_runtime': 'auto', 'domains': [], 'enable_plugins': [], 'encrypt_key': '', @@ -72,7 +94,6 @@ 'list_options': False, 'log_size': 0, 'map_file': '/etc/sos/cleaner/default_mapping', - 'master': '', 'primary': '', 'namespaces': None, 'nodes': [], @@ -101,6 +122,7 @@ 'ssh_port': 22, 'ssh_user': 'root', 'timeout': 600, + 'transport': 'auto', 'verify': False, 'usernames': [], 'upload': False, @@ -118,7 +140,7 @@ os.umask(0o77) self.client_list = [] self.node_list = [] - self.master = False + self.primary = False self.retrieved = 0 self.cluster = None self.cluster_type = None @@ -155,7 +177,6 @@ try: self.parse_node_strings() self.parse_cluster_options() - self._check_for_control_persist() self.log_debug('Executing %s' % ' '.join(s for s in sys.argv)) self.log_debug("Found cluster profiles: %s" % self.clusters.keys()) @@ -178,15 +199,17 @@ supported_clusters[cluster[0]] = cluster[1](self.commons) return supported_clusters - def _load_modules(self, package, submod): + @classmethod + def _load_modules(cls, package, submod): """Helper to import cluster and host types""" modules = [] for path in package.__path__: if os.path.isdir(path): - modules.extend(self._find_modules_in_path(path, submod)) + modules.extend(cls._find_modules_in_path(path, submod)) return modules - def _find_modules_in_path(self, path, modulename): + @classmethod + def _find_modules_in_path(cls, path, modulename): """Given a path and a module name, find everything that can be imported and then import it @@ -205,9 +228,10 @@ continue fname, ext = os.path.splitext(pyfile) modname = 'sos.collector.%s.%s' % (modulename, fname) - modules.extend(self._import_modules(modname)) + modules.extend(cls._import_modules(modname)) return modules + @classmethod def _import_modules(self, modname): """Import and return all found classes in a module""" mod_short_name = modname.split('.')[2] @@ -271,6 +295,9 @@ sos_grp.add_argument('--chroot', default='', choices=['auto', 'always', 'never'], help="chroot executed commands to SYSROOT") + sos_grp.add_argument("--container-runtime", default="auto", + help="Default container runtime to use for " + "collections. 'auto' for policy control.") sos_grp.add_argument('-e', '--enable-plugins', action="extend", help='Enable specific plugins for sosreport') sos_grp.add_argument('-k', '--plugin-option', '--plugopts', @@ -350,8 +377,6 @@ help='List options available for profiles') collect_grp.add_argument('--label', help='Assign a label to the archives') - collect_grp.add_argument('--master', - help='DEPRECATED: Specify a master node') collect_grp.add_argument('--primary', '--manager', '--controller', dest='primary', default='', help='Specify a primary node for cluster ' @@ -384,6 +409,8 @@ help='Specify an SSH user. Default root') collect_grp.add_argument('--timeout', type=int, required=False, help='Timeout for sosreport on each node.') + collect_grp.add_argument('--transport', default='auto', type=str, + help='Remote connection transport to use') collect_grp.add_argument("--upload", action="store_true", default=False, help="Upload archive to a policy-default " @@ -415,7 +442,7 @@ cleaner_grp.add_argument('--clean', '--cleaner', '--mask', dest='clean', default=False, action='store_true', - help='Obfuscate sensistive information') + help='Obfuscate sensitive information') cleaner_grp.add_argument('--keep-binary-files', default=False, action='store_true', dest='keep_binary_files', help='Keep unprocessable binary files in the ' @@ -440,35 +467,30 @@ action='extend', help='List of usernames to obfuscate') - def _check_for_control_persist(self): - """Checks to see if the local system supported SSH ControlPersist. - - ControlPersist allows OpenSSH to keep a single open connection to a - remote host rather than building a new session each time. This is the - same feature that Ansible uses in place of paramiko, which we have a - need to drop in sos-collector. - - This check relies on feedback from the ssh binary. The command being - run should always generate stderr output, but depending on what that - output reads we can determine if ControlPersist is supported or not. - - For our purposes, a host that does not support ControlPersist is not - able to run sos-collector. - - Returns - True if ControlPersist is supported, else raise Exception. - """ - ssh_cmd = ['ssh', '-o', 'ControlPersist'] - cmd = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = cmd.communicate() - err = err.decode('utf-8') - if 'Bad configuration option' in err or 'Usage:' in err: - raise ControlPersistUnsupportedException - return True + @classmethod + def display_help(cls, section): + section.set_title('SoS Collect Detailed Help') + section.add_text(cls.__doc__) + + hsections = { + 'collect.clusters': 'Information on cluster profiles', + 'collect.clusters.$cluster': 'Specific profile information', + 'collect.transports': 'Information on how connections are made', + 'collect.transports.$transport': 'Specific transport information' + } + section.add_text( + 'The following help sections may be of further interest:\n' + ) + for hsec in hsections: + section.add_text( + "{:>8}{:<40}{:<30}".format(' ', bold(hsec), hsections[hsec]), + newline=False + ) def exit(self, msg, error=1): """Used to safely terminate if sos-collector encounters an error""" + if self.cluster: + self.cluster.cleanup() self.log_error(msg) try: self.close_all_connections() @@ -485,7 +507,7 @@ 'cmdlineopts': self.opts, 'need_sudo': True if self.opts.ssh_user != 'root' else False, 'tmpdir': self.tmpdir, - 'hostlen': len(self.opts.master) or len(self.hostname), + 'hostlen': max(len(self.opts.primary), len(self.hostname)), 'policy': self.policy } @@ -656,7 +678,7 @@ on the commandline to point to one existing anywhere on the system Host groups define a list of nodes and/or regexes and optionally the - master and cluster-type options. + primary and cluster-type options. """ grp = self.opts.group paths = [ @@ -677,7 +699,7 @@ with open(fname, 'r') as hf: _group = json.load(hf) - for key in ['master', 'primary', 'cluster_type']: + for key in ['primary', 'cluster_type']: if _group[key]: self.log_debug("Setting option '%s' to '%s' per host group" % (key, _group[key])) @@ -691,12 +713,12 @@ Saves the results of this run of sos-collector to a host group file on the system so it can be used later on. - The host group will save the options master, cluster_type, and nodes + The host group will save the options primary, cluster_type, and nodes as determined by sos-collector prior to execution of sosreports. """ cfg = { 'name': self.opts.save_group, - 'primary': self.opts.master, + 'primary': self.opts.primary, 'cluster_type': self.cluster.cluster_type[0], 'nodes': [n for n in self.node_list] } @@ -722,7 +744,7 @@ self.ui_log.info(self._fmt_msg(msg)) if ((self.opts.password or (self.opts.password_per_node and - self.opts.master)) + self.opts.primary)) and not self.opts.batch): self.log_debug('password specified, not using SSH keys') msg = ('Provide the SSH password for user %s: ' @@ -769,8 +791,8 @@ self.policy.pre_work() - if self.opts.master: - self.connect_to_master() + if self.opts.primary: + self.connect_to_primary() self.opts.no_local = True else: try: @@ -797,9 +819,9 @@ self.ui_log.info(skip_local_msg) can_run_local = False self.opts.no_local = True - self.master = SosNode('localhost', self.commons, - local_sudo=local_sudo, - load_facts=can_run_local) + self.primary = SosNode('localhost', self.commons, + local_sudo=local_sudo, + load_facts=can_run_local) except Exception as err: self.log_debug("Unable to determine local installation: %s" % err) @@ -807,11 +829,11 @@ '--no-local option if localhost should not be ' 'included.\nAborting...\n', 1) - self.collect_md.add_field('primary', self.master.address) + self.collect_md.add_field('primary', self.primary.address) self.collect_md.add_section('nodes') - self.collect_md.nodes.add_section(self.master.address) - self.master.set_node_manifest(getattr(self.collect_md.nodes, - self.master.address)) + self.collect_md.nodes.add_section(self.primary.address) + self.primary.set_node_manifest(getattr(self.collect_md.nodes, + self.primary.address)) if self.opts.cluster_type: if self.opts.cluster_type == 'none': @@ -819,7 +841,7 @@ else: self.cluster = self.clusters[self.opts.cluster_type] self.cluster_type = self.opts.cluster_type - self.cluster.master = self.master + self.cluster.primary = self.primary else: self.determine_cluster() @@ -835,7 +857,9 @@ self.cluster_type = 'none' self.collect_md.add_field('cluster_type', self.cluster_type) if self.cluster: - self.master.cluster = self.cluster + self.primary.cluster = self.cluster + if self.opts.transport == 'auto': + self.opts.transport = self.cluster.set_transport_type() self.cluster.setup() if self.cluster.cluster_ssh_key: if not self.opts.ssh_key: @@ -858,15 +882,15 @@ """ self.ui_log.info('') - if not self.node_list and not self.master.connected: + if not self.node_list and not self.primary.connected: self.exit('No nodes were detected, or nodes do not have sos ' 'installed.\nAborting...') self.ui_log.info('The following is a list of nodes to collect from:') - if self.master.connected and self.master.hostname is not None: - if not (self.master.local and self.opts.no_local): + if self.primary.connected and self.primary.hostname is not None: + if not (self.primary.local and self.opts.no_local): self.ui_log.info('\t%-*s' % (self.commons['hostlen'], - self.master.hostname)) + self.primary.hostname)) for node in sorted(self.node_list): self.ui_log.info("\t%-*s" % (self.commons['hostlen'], node)) @@ -879,6 +903,8 @@ self.ui_log.info("") except KeyboardInterrupt: self.exit("Exiting on user cancel", 130) + except Exception as e: + self.exit(repr(e), 1) def configure_sos_cmd(self): """Configures the sosreport command that is run on the nodes""" @@ -919,17 +945,17 @@ self.commons['sos_cmd'] = self.sos_cmd self.collect_md.add_field('initial_sos_cmd', self.sos_cmd) - def connect_to_master(self): - """If run with --master, we will run cluster checks again that + def connect_to_primary(self): + """If run with --primary, we will run cluster checks again that instead of the localhost. """ try: - self.master = SosNode(self.opts.master, self.commons) + self.primary = SosNode(self.opts.primary, self.commons) self.ui_log.info('Connected to %s, determining cluster type...' - % self.opts.master) + % self.opts.primary) except Exception as e: - self.log_debug('Failed to connect to master: %s' % e) - self.exit('Could not connect to master node. Aborting...', 1) + self.log_debug('Failed to connect to primary node: %s' % e) + self.exit('Could not connect to primary node. Aborting...', 1) def determine_cluster(self): """This sets the cluster type and loads that cluster's cluster. @@ -943,7 +969,7 @@ checks = list(self.clusters.values()) for cluster in self.clusters.values(): checks.remove(cluster) - cluster.master = self.master + cluster.primary = self.primary if cluster.check_enabled(): cname = cluster.__class__.__name__ self.log_debug("Installation matches %s, checking for layered " @@ -954,7 +980,7 @@ self.log_debug("Layered profile %s found. " "Checking installation" % rname) - remaining.master = self.master + remaining.primary = self.primary if remaining.check_enabled(): self.log_debug("Installation matches both layered " "profile %s and base profile %s, " @@ -978,18 +1004,18 @@ return [] def reduce_node_list(self): - """Reduce duplicate entries of the localhost and/or master node + """Reduce duplicate entries of the localhost and/or primary node if applicable""" if (self.hostname in self.node_list and self.opts.no_local): self.node_list.remove(self.hostname) for i in self.ip_addrs: if i in self.node_list: self.node_list.remove(i) - # remove the master node from the list, since we already have + # remove the primary node from the list, since we already have # an open session to it. - if self.master: + if self.primary: for n in self.node_list: - if n == self.master.hostname or n == self.opts.master: + if n == self.primary.hostname or n == self.opts.primary: self.node_list.remove(n) self.node_list = list(set(n for n in self.node_list if n)) self.log_debug('Node list reduced to %s' % self.node_list) @@ -1010,9 +1036,9 @@ def get_nodes(self): """ Sets the list of nodes to collect sosreports from """ - if not self.master and not self.cluster: + if not self.primary and not self.cluster: msg = ('Could not determine a cluster type and no list of ' - 'nodes or master node was provided.\nAborting...' + 'nodes or primary node was provided.\nAborting...' ) self.exit(msg) @@ -1041,7 +1067,7 @@ self.log_debug("Force adding %s to node list" % node) self.node_list.append(node) - if not self.master: + if not self.primary: host = self.hostname.split('.')[0] # trust the local hostname before the node report from cluster for node in self.node_list: @@ -1050,9 +1076,10 @@ self.node_list.append(self.hostname) self.reduce_node_list() try: - self.commons['hostlen'] = len(max(self.node_list, key=len)) + _node_max = len(max(self.node_list, key=len)) + self.commons['hostlen'] = max(_node_max, self.commons['hostlen']) except (TypeError, ValueError): - self.commons['hostlen'] = len(self.opts.master) + pass def _connect_to_node(self, node): """Try to connect to the node, and if we can add to the client list to @@ -1071,8 +1098,9 @@ client.set_node_manifest(getattr(self.collect_md.nodes, node[0])) else: - client.close_ssh_session() + client.disconnect() except Exception: + # all exception logging is handled within SoSNode pass def intro(self): @@ -1080,12 +1108,11 @@ provided on the command line """ disclaimer = ("""\ -This utility is used to collect sosreports from multiple \ -nodes simultaneously. It uses OpenSSH's ControlPersist feature \ -to connect to nodes and run commands remotely. If your system \ -installation of OpenSSH is older than 5.6, please upgrade. +This utility is used to collect sos reports from multiple \ +nodes simultaneously. Remote connections are made and/or maintained \ +to those nodes via well-known transport protocols such as SSH. -An archive of sosreport tarballs collected from the nodes will be \ +An archive of sos report tarballs collected from the nodes will be \ generated in %s and may be provided to an appropriate support representative. The generated archive may contain data considered sensitive \ @@ -1099,12 +1126,6 @@ intro_msg = self._fmt_msg(disclaimer % self.tmpdir) self.ui_log.info(intro_msg) - if self.opts.master: - self.ui_log.info( - "NOTE: Use of '--master' is DEPRECATED and will be removed in " - "a future release.\nUse '--primary', '--manager', or " - "'--controller' instead.") - prompt = "\nPress ENTER to continue, or CTRL-C to quit\n" if not self.opts.batch: try: @@ -1112,11 +1133,8 @@ self.ui_log.info("") except KeyboardInterrupt: self.exit("Exiting on user cancel", 130) - - if not self.opts.case_id and not self.opts.batch: - msg = 'Optionally, please enter the case id you are collecting ' \ - 'reports for: ' - self.opts.case_id = input(msg) + except Exception as e: + self._exit(1, e) def execute(self): if self.opts.list_options: @@ -1126,12 +1144,6 @@ self.intro() - if self.opts.primary: - # for now, use the new option name and simply override the existing - # value that the rest of the component references. Full conversion - # of master -> primary is a 4.3 item. - self.opts.master = self.opts.primary - self.configure_sos_cmd() self.prep() self.display_nodes() @@ -1142,16 +1154,17 @@ self.archive.makedirs('sos_logs', 0o755) self.collect() + self.cluster.cleanup() self.cleanup() def collect(self): """ For each node, start a collection thread and then tar all collected sosreports """ - if self.master.connected: - self.client_list.append(self.master) + if self.primary.connected: + self.client_list.append(self.primary) self.ui_log.info("\nConnecting to nodes...") - filters = [self.master.address, self.master.hostname] + filters = [self.primary.address, self.primary.hostname] nodes = [(n, None) for n in self.node_list if n not in filters] if self.opts.password_per_node: @@ -1200,16 +1213,18 @@ pool.shutdown(wait=True) except KeyboardInterrupt: self.log_error('Exiting on user cancel\n') + self.cluster.cleanup() os._exit(130) except Exception as err: self.log_error('Could not connect to nodes: %s' % err) + self.cluster.cleanup() os._exit(1) if hasattr(self.cluster, 'run_extra_cmd'): - self.ui_log.info('Collecting additional data from master node...') + self.ui_log.info('Collecting additional data from primary node...') files = self.cluster._run_extra_cmd() if files: - self.master.collect_extra_cmd(files) + self.primary.collect_extra_cmd(files) msg = '\nSuccessfully captured %s of %s sosreports' self.log_info(msg % (self.retrieved, self.report_num)) self.close_all_connections() @@ -1219,7 +1234,7 @@ msg = 'No sosreports were collected, nothing to archive...' self.exit(msg, 1) - if self.opts.upload and self.get_upload_url(): + if self.opts.upload and self.policy.get_upload_url(): try: self.policy.upload_archive(arc_name) self.ui_log.info("Uploaded archive successfully") @@ -1250,10 +1265,11 @@ self.log_error("Error running sosreport: %s" % err) def close_all_connections(self): - """Close all ssh sessions for nodes""" + """Close all sessions for nodes""" for client in self.client_list: - self.log_debug('Closing SSH connection to %s' % client.address) - client.close_ssh_session() + if client.connected: + self.log_debug('Closing connection to %s' % client.address) + client.disconnect() def create_cluster_archive(self): """Calls for creation of tar archive then cleans up the temporary diff -Nru sosreport-4.2/sos/collector/sosnode.py sosreport-4.3/sos/collector/sosnode.py --- sosreport-4.2/sos/collector/sosnode.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/collector/sosnode.py 2022-02-15 04:20:20.000000000 +0000 @@ -12,23 +12,25 @@ import inspect import logging import os -import pexpect import re -import shutil from distutils.version import LooseVersion from pipes import quote from sos.policies import load from sos.policies.init_systems import InitSystem -from sos.collector.exceptions import (InvalidPasswordException, - TimeoutPasswordAuthException, - PasswordRequestException, - AuthPermissionDeniedException, +from sos.collector.transports.control_persist import SSHControlPersist +from sos.collector.transports.local import LocalTransport +from sos.collector.transports.oc import OCTransport +from sos.collector.exceptions import (CommandTimeoutException, ConnectionException, - CommandTimeoutException, - ConnectionTimeoutException, - ControlSocketMissingException, - UnsupportedHostException) + UnsupportedHostException, + InvalidTransportException) + +TRANSPORTS = { + 'local': LocalTransport, + 'control_persist': SSHControlPersist, + 'oc': OCTransport +} class SosNode(): @@ -67,34 +69,25 @@ 'sos_cmd': commons['sos_cmd'] } self.sos_bin = 'sosreport' - filt = ['localhost', '127.0.0.1'] self.soslog = logging.getLogger('sos') self.ui_log = logging.getLogger('sos_ui') - self.control_path = ("%s/.sos-collector-%s" - % (self.tmpdir, self.address)) - self.ssh_cmd = self._create_ssh_command() - if self.address not in filt: - try: - self.connected = self._create_ssh_session() - except Exception as err: - self.log_error('Unable to open SSH session: %s' % err) - raise - else: - self.connected = True - self.local = True - self.need_sudo = os.getuid() != 0 + self._transport = self._load_remote_transport(commons) + try: + self._transport.connect(self._password) + except Exception as err: + self.log_error('Unable to open remote session: %s' % err) + raise # load the host policy now, even if we don't want to load further # host information. This is necessary if we're running locally on the - # cluster master but do not want a local report as we still need to do + # cluster primary but do not want a local report as we still need to do # package checks in that instance self.host = self.determine_host_policy() - self.get_hostname() + self.hostname = self._transport.hostname if self.local and self.opts.no_local: load_facts = False if self.connected and load_facts: if not self.host: - self.connected = False - self.close_ssh_session() + self._transport.disconnect() return None if self.local: if self.check_in_container(): @@ -103,11 +96,34 @@ self.create_sos_container() self._load_sos_info() - def _create_ssh_command(self): - """Build the complete ssh command for this node""" - cmd = "ssh -oControlPath=%s " % self.control_path - cmd += "%s@%s " % (self.opts.ssh_user, self.address) - return cmd + @property + def connected(self): + if self._transport: + return self._transport.connected + # if no transport, we're running locally + return True + + def disconnect(self): + """Wrapper to close the remote session via our transport agent + """ + self._transport.disconnect() + + def _load_remote_transport(self, commons): + """Determine the type of remote transport to load for this node, then + return an instantiated instance of that transport + """ + if self.address in ['localhost', '127.0.0.1']: + self.local = True + return LocalTransport(self.address, commons) + elif self.opts.transport in TRANSPORTS.keys(): + return TRANSPORTS[self.opts.transport](self.address, commons) + elif self.opts.transport != 'auto': + self.log_error( + "Connection failed: unknown or unsupported transport %s" + % self.opts.transport + ) + raise InvalidTransportException(self.opts.transport) + return SSHControlPersist(self.address, commons) def _fmt_msg(self, msg): return '{:<{}} : {}'.format(self._hostname, self.hostlen + 1, msg) @@ -135,6 +151,7 @@ self.manifest.add_field('policy', self.host.distro) self.manifest.add_field('sos_version', self.sos_info['version']) self.manifest.add_field('final_sos_command', '') + self.manifest.add_field('transport', self._transport.name) def check_in_container(self): """ @@ -160,13 +177,13 @@ res = self.run_command(cmd, need_root=True) if res['status'] in [0, 125]: if res['status'] == 125: - if 'unable to retrieve auth token' in res['stdout']: + if 'unable to retrieve auth token' in res['output']: self.log_error( "Could not pull image. Provide either a username " "and password or authfile" ) raise Exception - elif 'unknown: Not found' in res['stdout']: + elif 'unknown: Not found' in res['output']: self.log_error('Specified image not found on registry') raise Exception # 'name exists' with code 125 means the container was @@ -181,11 +198,11 @@ return True else: self.log_error("Could not start container after create: %s" - % ret['stdout']) + % ret['output']) raise Exception else: self.log_error("Could not create container on host: %s" - % res['stdout']) + % res['output']) raise Exception def get_container_auth(self): @@ -204,18 +221,11 @@ def file_exists(self, fname, need_root=False): """Checks for the presence of fname on the remote node""" - if not self.local: - try: - res = self.run_command("stat %s" % fname, need_root=need_root) - return res['status'] == 0 - except Exception: - return False - else: - try: - os.stat(fname) - return True - except Exception: - return False + try: + res = self.run_command("stat %s" % fname, need_root=need_root) + return res['status'] == 0 + except Exception: + return False @property def _hostname(self): @@ -223,18 +233,6 @@ return self.hostname return self.address - @property - def control_socket_exists(self): - """Check if the SSH control socket exists - - The control socket is automatically removed by the SSH daemon in the - event that the last connection to the node was greater than the timeout - set by the ControlPersist option. This can happen for us if we are - collecting from a large number of nodes, and the timeout expires before - we start collection. - """ - return os.path.exists(self.control_path) - def _sanitize_log_msg(self, msg): """Attempts to obfuscate sensitive information in log messages such as passwords""" @@ -264,12 +262,6 @@ msg = '[%s:%s] %s' % (self._hostname, caller, msg) self.soslog.debug(msg) - def get_hostname(self): - """Get the node's hostname""" - sout = self.run_command('hostname') - self.hostname = sout['stdout'].strip() - self.log_info('Hostname set to %s' % self.hostname) - def _format_cmd(self, cmd): """If we need to provide a sudo or root password to a command, then here we prefix the command with the correct bits @@ -280,19 +272,6 @@ return "sudo -S %s" % cmd return cmd - def _fmt_output(self, output=None, rc=0): - """Formats the returned output from a command into a dict""" - if rc == 0: - stdout = output - stderr = '' - else: - stdout = '' - stderr = output - res = {'status': rc, - 'stdout': stdout, - 'stderr': stderr} - return res - def _load_sos_info(self): """Queries the node for information about the installed version of sos """ @@ -306,7 +285,7 @@ pkgs = self.run_command(self.host.container_version_command, use_container=True, need_root=True) if pkgs['status'] == 0: - ver = pkgs['stdout'].strip().split('-')[1] + ver = pkgs['output'].strip().split('-')[1] if ver: self.sos_info['version'] = ver else: @@ -314,25 +293,28 @@ if self.sos_info['version']: self.log_info('sos version is %s' % self.sos_info['version']) else: - if not self.address == self.opts.master: - # in the case where the 'master' enumerates nodes but is not + if not self.address == self.opts.primary: + # in the case where the 'primary' enumerates nodes but is not # intended for collection (bastions), don't worry about sos not # being present self.log_error('sos is not installed on this node') self.connected = False return False - cmd = 'sosreport -l' + # sos-4.0 changes the binary + if self.check_sos_version('4.0'): + self.sos_bin = 'sos report' + cmd = "%s -l" % self.sos_bin sosinfo = self.run_command(cmd, use_container=True, need_root=True) if sosinfo['status'] == 0: - self._load_sos_plugins(sosinfo['stdout']) + self._load_sos_plugins(sosinfo['output']) if self.check_sos_version('3.6'): self._load_sos_presets() def _load_sos_presets(self): - cmd = 'sosreport --list-presets' + cmd = '%s --list-presets' % self.sos_bin res = self.run_command(cmd, use_container=True, need_root=True) if res['status'] == 0: - for line in res['stdout'].splitlines(): + for line in res['output'].splitlines(): if line.strip().startswith('name:'): pname = line.split('name:')[1].strip() self.sos_info['presets'].append(pname) @@ -372,21 +354,7 @@ """Reads the specified file and returns the contents""" try: self.log_info("Reading file %s" % to_read) - if not self.local: - res = self.run_command("cat %s" % to_read, timeout=5) - if res['status'] == 0: - return res['stdout'] - else: - if 'No such file' in res['stdout']: - self.log_debug("File %s does not exist on node" - % to_read) - else: - self.log_error("Error reading %s: %s" % - (to_read, res['stdout'].split(':')[1:])) - return '' - else: - with open(to_read, 'r') as rfile: - return rfile.read() + return self._transport.read_file(to_read) except Exception as err: self.log_error("Exception while reading %s: %s" % (to_read, err)) return '' @@ -400,7 +368,8 @@ % self.commons['policy'].distro) return self.commons['policy'] host = load(cache={}, sysroot=self.opts.sysroot, init=InitSystem(), - probe_runtime=True, remote_exec=self.ssh_cmd, + probe_runtime=True, + remote_exec=self._transport.remote_exec, remote_check=self.read_file('/etc/os-release')) if host: self.log_info("loaded policy %s for host" % host.distro) @@ -413,7 +382,8 @@ given ver. This means that if the installed version is greater than ver, this will still return True """ - return LooseVersion(self.sos_info['version']) >= ver + return self.sos_info['version'] is not None and \ + LooseVersion(self.sos_info['version']) >= ver def is_installed(self, pkg): """Checks if a given package is installed on the node""" @@ -422,7 +392,7 @@ return self.host.package_manager.pkg_by_name(pkg) is not None def run_command(self, cmd, timeout=180, get_pty=False, need_root=False, - force_local=False, use_container=False, env=None): + use_container=False, env=None): """Runs a given cmd, either via the SSH session or locally Arguments: @@ -433,58 +403,35 @@ need_root - if a command requires root privileges, setting this to True tells sos-collector to format the command with sudo or su - as appropriate and to input the password - force_local - force a command to run locally. Mainly used for scp. use_container - Run this command in a container *IF* the host is containerized """ - if not self.control_socket_exists and not self.local: - self.log_debug('Control socket does not exist, attempting to ' - 're-create') + if not self.connected and not self.local: + self.log_debug('Node is disconnected, attempting to reconnect') try: - _sock = self._create_ssh_session() - if not _sock: - self.log_debug('Failed to re-create control socket') - raise ControlSocketMissingException + reconnected = self._transport.reconnect(self._password) + if not reconnected: + self.log_debug('Failed to reconnect to node') + raise ConnectionException except Exception as err: - self.log_error('Cannot run command: control socket does not ' - 'exist') - self.log_debug("Error while trying to create new SSH control " - "socket: %s" % err) + self.log_debug("Error while trying to reconnect: %s" % err) raise if use_container and self.host.containerized: cmd = self.host.format_container_command(cmd) if need_root: - get_pty = True cmd = self._format_cmd(cmd) - self.log_debug('Running command %s' % cmd) + if 'atomic' in cmd: get_pty = True - if not self.local and not force_local: - cmd = "%s %s" % (self.ssh_cmd, quote(cmd)) - else: - if get_pty: - cmd = "/bin/bash -c %s" % quote(cmd) + if env: _cmd_env = self.env_vars env = _cmd_env.update(env) - res = pexpect.spawn(cmd, encoding='utf-8', env=env) - if need_root: - if self.need_sudo: - res.sendline(self.opts.sudo_pw) - if self.opts.become_root: - res.sendline(self.opts.root_password) - output = res.expect([pexpect.EOF, pexpect.TIMEOUT], - timeout=timeout) - if output == 0: - out = res.before - res.close() - rc = res.exitstatus - return {'status': rc, 'stdout': out} - elif output == 1: - raise CommandTimeoutException(cmd) + return self._transport.run_command(cmd, timeout, need_root, env, + get_pty) def sosreport(self): - """Run a sosreport on the node, then collect it""" + """Run an sos report on the node, then collect it""" try: path = self.execute_sos_command() if path: @@ -497,109 +444,6 @@ pass self.cleanup() - def _create_ssh_session(self): - """ - Using ControlPersist, create the initial connection to the node. - - This will generate an OpenSSH ControlPersist socket within the tmp - directory created or specified for sos-collector to use. - - At most, we will wait 30 seconds for a connection. This involves a 15 - second wait for the initial connection attempt, and a subsequent 15 - second wait for a response when we supply a password. - - Since we connect to nodes in parallel (using the --threads value), this - means that the time between 'Connecting to nodes...' and 'Beginning - collection of sosreports' that users see can be up to an amount of time - equal to 30*(num_nodes/threads) seconds. - - Returns - True if session is successfully opened, else raise Exception - """ - # Don't use self.ssh_cmd here as we need to add a few additional - # parameters to establish the initial connection - self.log_info('Opening SSH session to create control socket') - connected = False - ssh_key = '' - ssh_port = '' - if self.opts.ssh_port != 22: - ssh_port = "-p%s " % self.opts.ssh_port - if self.opts.ssh_key: - ssh_key = "-i%s" % self.opts.ssh_key - cmd = ("ssh %s %s -oControlPersist=600 -oControlMaster=auto " - "-oStrictHostKeyChecking=no -oControlPath=%s %s@%s " - "\"echo Connected\"" % (ssh_key, - ssh_port, - self.control_path, - self.opts.ssh_user, - self.address)) - res = pexpect.spawn(cmd, encoding='utf-8') - - connect_expects = [ - u'Connected', - u'password:', - u'.*Permission denied.*', - u'.* port .*: No route to host', - u'.*Could not resolve hostname.*', - pexpect.TIMEOUT - ] - - index = res.expect(connect_expects, timeout=15) - - if index == 0: - connected = True - elif index == 1: - if self._password: - pass_expects = [ - u'Connected', - u'Permission denied, please try again.', - pexpect.TIMEOUT - ] - res.sendline(self._password) - pass_index = res.expect(pass_expects, timeout=15) - if pass_index == 0: - connected = True - elif pass_index == 1: - # Note that we do not get an exitstatus here, so matching - # this line means an invalid password will be reported for - # both invalid passwords and invalid user names - raise InvalidPasswordException - elif pass_index == 2: - raise TimeoutPasswordAuthException - else: - raise PasswordRequestException - elif index == 2: - raise AuthPermissionDeniedException - elif index == 3: - raise ConnectionException(self.address, self.opts.ssh_port) - elif index == 4: - raise ConnectionException(self.address) - elif index == 5: - raise ConnectionTimeoutException - else: - raise Exception("Unknown error, client returned %s" % res.before) - if connected: - self.log_debug("Successfully created control socket at %s" - % self.control_path) - return True - return False - - def close_ssh_session(self): - """Remove the control socket to effectively terminate the session""" - if self.local: - return True - try: - res = self.run_command("rm -f %s" % self.control_path, - force_local=True) - if res['status'] == 0: - return True - self.log_error("Could not remove ControlPath %s: %s" - % (self.control_path, res['stdout'])) - return False - except Exception as e: - self.log_error('Error closing SSH session: %s' % e) - return False - def _preset_exists(self, preset): """Verifies if the given preset exists on the node""" return preset in self.sos_info['presets'] @@ -646,8 +490,8 @@ self.cluster = cluster def update_cmd_from_cluster(self): - """This is used to modify the sosreport command run on the nodes. - By default, sosreport is run without any options, using this will + """This is used to modify the sos report command run on the nodes. + By default, sos report is run without any options, using this will allow the profile to specify what plugins to run or not and what options to use. @@ -672,10 +516,10 @@ self.cluster.sos_plugin_options[opt]) self.plugopts.append(option) - # set master-only options - if self.cluster.check_node_is_master(self): + # set primary-only options + if self.cluster.check_node_is_primary(self): with self.cluster.lock: - self.cluster.set_master_options(self) + self.cluster.set_primary_options(self) else: with self.cluster.lock: self.cluster.set_node_options(self) @@ -727,10 +571,6 @@ if self.opts.since: sos_opts.append('--since=%s' % quote(self.opts.since)) - # sos-4.0 changes the binary - if self.check_sos_version('4.0'): - self.sos_bin = 'sos report' - if self.check_sos_version('4.1'): if self.opts.skip_commands: sos_opts.append( @@ -747,6 +587,12 @@ sos_opts.append('--cmd-timeout=%s' % quote(str(self.opts.cmd_timeout))) + if self.check_sos_version('4.3'): + if self.opts.container_runtime != 'auto': + sos_opts.append( + "--container-runtime=%s" % self.opts.container_runtime + ) + self.update_cmd_from_cluster() sos_cmd = sos_cmd.replace( @@ -811,7 +657,7 @@ self.manifest.add_field('final_sos_command', self.sos_cmd) def determine_sos_label(self): - """Determine what, if any, label should be added to the sosreport""" + """Determine what, if any, label should be added to the sos report""" label = '' label += self.cluster.get_node_label(self) @@ -822,7 +668,7 @@ if not label: return None - self.log_debug('Label for sosreport set to %s' % label) + self.log_debug('Label for sos report set to %s' % label) if self.check_sos_version('3.6'): lcmd = '--label' else: @@ -844,20 +690,20 @@ def determine_sos_error(self, rc, stdout): if rc == -1: - return 'sosreport process received SIGKILL on node' + return 'sos report process received SIGKILL on node' if rc == 1: if 'sudo' in stdout: return 'sudo attempt failed' if rc == 127: - return 'sosreport terminated unexpectedly. Check disk space' + return 'sos report terminated unexpectedly. Check disk space' if len(stdout) > 0: return stdout.split('\n')[0:1] else: return 'sos exited with code %s' % rc def execute_sos_command(self): - """Run sosreport and capture the resulting file path""" - self.ui_msg('Generating sosreport...') + """Run sos report and capture the resulting file path""" + self.ui_msg('Generating sos report...') try: path = False checksum = False @@ -867,7 +713,7 @@ use_container=True, env=self.sos_env_vars) if res['status'] == 0: - for line in res['stdout'].splitlines(): + for line in res['output'].splitlines(): if fnmatch.fnmatch(line, '*sosreport-*tar*'): path = line.strip() if line.startswith((" sha256\t", " md5\t")): @@ -875,7 +721,7 @@ elif line.startswith("The checksum is: "): checksum = line.split()[3] - if checksum is not None: + if checksum: self.manifest.add_field('checksum', checksum) if len(checksum) == 32: self.manifest.add_field('checksum_type', 'md5') @@ -883,46 +729,34 @@ self.manifest.add_field('checksum_type', 'sha256') else: self.manifest.add_field('checksum_type', 'unknown') + else: + self.manifest.add_field('checksum_type', 'unknown') else: - err = self.determine_sos_error(res['status'], res['stdout']) - self.log_debug("Error running sosreport. rc = %s msg = %s" - % (res['status'], res['stdout'] or - res['stderr'])) + err = self.determine_sos_error(res['status'], res['output']) + self.log_debug("Error running sos report. rc = %s msg = %s" + % (res['status'], res['output'])) raise Exception(err) return path except CommandTimeoutException: self.log_error('Timeout exceeded') raise except Exception as e: - self.log_error('Error running sosreport: %s' % e) + self.log_error('Error running sos report: %s' % e) raise def retrieve_file(self, path): """Copies the specified file from the host to our temp dir""" destdir = self.tmpdir + '/' - dest = destdir + path.split('/')[-1] + dest = os.path.join(destdir, path.split('/')[-1]) try: - if not self.local: - if self.file_exists(path): - self.log_info("Copying remote %s to local %s" % - (path, destdir)) - cmd = "/usr/bin/scp -oControlPath=%s %s@%s:%s %s" % ( - self.control_path, - self.opts.ssh_user, - self.address, - path, - destdir - ) - res = self.run_command(cmd, force_local=True) - return res['status'] == 0 - else: - self.log_debug("Attempting to copy remote file %s, but it " - "does not exist on filesystem" % path) - return False + if self.file_exists(path): + self.log_info("Copying remote %s to local %s" % + (path, destdir)) + return self._transport.retrieve_file(path, dest) else: - self.log_debug("Moving %s to %s" % (path, destdir)) - shutil.copy(path, dest) - return True + self.log_debug("Attempting to copy remote file %s, but it " + "does not exist on filesystem" % path) + return False except Exception as err: self.log_debug("Failed to retrieve %s: %s" % (path, err)) return False @@ -933,7 +767,7 @@ """ path = ''.join(path.split()) try: - if len(path) <= 2: # ensure we have a non '/' path + if len(path.split('/')) <= 2: # ensure we have a non '/' path self.log_debug("Refusing to remove path %s: appears to be " "incorrect and possibly dangerous" % path) return False @@ -959,16 +793,20 @@ except Exception: self.log_error('Failed to make archive readable') return False - self.soslog.info('Retrieving sosreport from %s' % self.address) - self.ui_msg('Retrieving sosreport...') - ret = self.retrieve_file(self.sos_path) + self.log_info('Retrieving sos report from %s' % self.address) + self.ui_msg('Retrieving sos report...') + try: + ret = self.retrieve_file(self.sos_path) + except Exception as err: + self.log_error(err) + return False if ret: - self.ui_msg('Successfully collected sosreport') + self.ui_msg('Successfully collected sos report') self.file_list.append(self.sos_path.split('/')[-1]) + return True else: - self.log_error('Failed to retrieve sosreport') - raise SystemExit - return True + self.ui_msg('Failed to retrieve sos report') + return False else: # sos sometimes fails but still returns a 0 exit code if self.stderr.read(): @@ -976,30 +814,32 @@ else: e = [x.strip() for x in self.stdout.readlines() if x.strip][-1] self.soslog.error( - 'Failed to run sosreport on %s: %s' % (self.address, e)) - self.log_error('Failed to run sosreport. %s' % e) + 'Failed to run sos report on %s: %s' % (self.address, e)) + self.log_error('Failed to run sos report. %s' % e) return False def remove_sos_archive(self): """Remove the sosreport archive from the node, since we have collected it and it would be wasted space otherwise""" - if self.sos_path is None: + if self.sos_path is None or self.local: + # local transport moves the archive rather than copies it, so there + # is no archive at the original location to remove return if 'sosreport' not in self.sos_path: - self.log_debug("Node sosreport path %s looks incorrect. Not " + self.log_debug("Node sos report path %s looks incorrect. Not " "attempting to remove path" % self.sos_path) return removed = self.remove_file(self.sos_path) if not removed: - self.log_error('Failed to remove sosreport') + self.log_error('Failed to remove sos report') def cleanup(self): """Remove the sos archive from the node once we have it locally""" self.remove_sos_archive() if self.sos_path: for ext in ['.sha256', '.md5']: - if os.path.isfile(self.sos_path + ext): - self.remove_file(self.sos_path + ext) + if self.remove_file(self.sos_path + ext): + break cleanup = self.host.set_cleanup_cmd() if cleanup: self.run_command(cleanup, need_root=True) @@ -1023,7 +863,7 @@ else: self.log_error("Unable to retrieve file %s" % filename) except Exception as e: - msg = 'Error collecting additional data from master: %s' % e + msg = 'Error collecting additional data from primary: %s' % e self.log_error(msg) def make_archive_readable(self, filepath): @@ -1040,3 +880,5 @@ msg = "Exception while making %s readable. Return code was %s" self.log_error(msg % (filepath, res['status'])) raise Exception + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/transports/control_persist.py sosreport-4.3/sos/collector/transports/control_persist.py --- sosreport-4.2/sos/collector/transports/control_persist.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/collector/transports/control_persist.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,207 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +import os +import pexpect +import subprocess + +from sos.collector.transports import RemoteTransport +from sos.collector.exceptions import (InvalidPasswordException, + TimeoutPasswordAuthException, + PasswordRequestException, + AuthPermissionDeniedException, + ConnectionException, + ConnectionTimeoutException, + ControlSocketMissingException, + ControlPersistUnsupportedException) +from sos.utilities import sos_get_command_output + + +class SSHControlPersist(RemoteTransport): + """ + A transport for collect that leverages OpenSSH's ControlPersist + functionality which uses control sockets to transparently keep a connection + open to the remote host without needing to rebuild the SSH connection for + each and every command executed on the node. + + This transport will by default assume the use of SSH keys, meaning keys + have already been distributed to target nodes. If this is not the case, + users will need to provide a password using the --password or + --password-per-node option, depending on if the password to connect to all + nodes is the same or not. Note that these options prevent the use of the + --batch option, as they require user input. + """ + + name = 'control_persist' + + def _check_for_control_persist(self): + """Checks to see if the local system supported SSH ControlPersist. + + ControlPersist allows OpenSSH to keep a single open connection to a + remote host rather than building a new session each time. This is the + same feature that Ansible uses in place of paramiko, which we have a + need to drop in sos-collector. + + This check relies on feedback from the ssh binary. The command being + run should always generate stderr output, but depending on what that + output reads we can determine if ControlPersist is supported or not. + + For our purposes, a host that does not support ControlPersist is not + able to run sos-collector. + + Returns + True if ControlPersist is supported, else raise Exception. + """ + ssh_cmd = ['ssh', '-o', 'ControlPersist'] + cmd = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = cmd.communicate() + err = err.decode('utf-8') + if 'Bad configuration option' in err or 'Usage:' in err: + raise ControlPersistUnsupportedException + return True + + def _connect(self, password=''): + """ + Using ControlPersist, create the initial connection to the node. + + This will generate an OpenSSH ControlPersist socket within the tmp + directory created or specified for sos-collector to use. + + At most, we will wait 30 seconds for a connection. This involves a 15 + second wait for the initial connection attempt, and a subsequent 15 + second wait for a response when we supply a password. + + Since we connect to nodes in parallel (using the --threads value), this + means that the time between 'Connecting to nodes...' and 'Beginning + collection of sosreports' that users see can be up to an amount of time + equal to 30*(num_nodes/threads) seconds. + + Returns + True if session is successfully opened, else raise Exception + """ + try: + self._check_for_control_persist() + except ControlPersistUnsupportedException: + self.log_error("OpenSSH ControlPersist is not locally supported. " + "Please update your OpenSSH installation.") + raise + self.log_info('Opening SSH session to create control socket') + self.control_path = ("%s/.sos-collector-%s" % (self.tmpdir, + self.address)) + self.ssh_cmd = '' + connected = False + ssh_key = '' + ssh_port = '' + if self.opts.ssh_port != 22: + ssh_port = "-p%s " % self.opts.ssh_port + if self.opts.ssh_key: + ssh_key = "-i%s" % self.opts.ssh_key + + cmd = ("ssh %s %s -oControlPersist=600 -oControlMaster=auto " + "-oStrictHostKeyChecking=no -oControlPath=%s %s@%s " + "\"echo Connected\"" % (ssh_key, + ssh_port, + self.control_path, + self.opts.ssh_user, + self.address)) + res = pexpect.spawn(cmd, encoding='utf-8') + + connect_expects = [ + u'Connected', + u'password:', + u'.*Permission denied.*', + u'.* port .*: No route to host', + u'.*Could not resolve hostname.*', + pexpect.TIMEOUT + ] + + index = res.expect(connect_expects, timeout=15) + + if index == 0: + connected = True + elif index == 1: + if password: + pass_expects = [ + u'Connected', + u'Permission denied, please try again.', + pexpect.TIMEOUT + ] + res.sendline(password) + pass_index = res.expect(pass_expects, timeout=15) + if pass_index == 0: + connected = True + elif pass_index == 1: + # Note that we do not get an exitstatus here, so matching + # this line means an invalid password will be reported for + # both invalid passwords and invalid user names + raise InvalidPasswordException + elif pass_index == 2: + raise TimeoutPasswordAuthException + else: + raise PasswordRequestException + elif index == 2: + raise AuthPermissionDeniedException + elif index == 3: + raise ConnectionException(self.address, self.opts.ssh_port) + elif index == 4: + raise ConnectionException(self.address) + elif index == 5: + raise ConnectionTimeoutException + else: + raise Exception("Unknown error, client returned %s" % res.before) + if connected: + if not os.path.exists(self.control_path): + raise ControlSocketMissingException + self.log_debug("Successfully created control socket at %s" + % self.control_path) + return True + return False + + def _disconnect(self): + if os.path.exists(self.control_path): + try: + os.remove(self.control_path) + return True + except Exception as err: + self.log_debug("Could not disconnect properly: %s" % err) + return False + self.log_debug("Control socket not present when attempting to " + "terminate session") + + @property + def connected(self): + """Check if the SSH control socket exists + + The control socket is automatically removed by the SSH daemon in the + event that the last connection to the node was greater than the timeout + set by the ControlPersist option. This can happen for us if we are + collecting from a large number of nodes, and the timeout expires before + we start collection. + """ + return os.path.exists(self.control_path) + + @property + def remote_exec(self): + if not self.ssh_cmd: + self.ssh_cmd = "ssh -oControlPath=%s %s@%s" % ( + self.control_path, self.opts.ssh_user, self.address + ) + return self.ssh_cmd + + def _retrieve_file(self, fname, dest): + cmd = "/usr/bin/scp -oControlPath=%s %s@%s:%s %s" % ( + self.control_path, self.opts.ssh_user, self.address, fname, dest + ) + res = sos_get_command_output(cmd) + return res['status'] == 0 + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/transports/__init__.py sosreport-4.3/sos/collector/transports/__init__.py --- sosreport-4.2/sos/collector/transports/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/collector/transports/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,398 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import inspect +import logging +import pexpect +import re + +from pipes import quote +from sos.collector.exceptions import (ConnectionException, + CommandTimeoutException) +from sos.utilities import bold + + +class RemoteTransport(): + """The base class used for defining supported remote transports to connect + to remote nodes in conjunction with `sos collect`. + + This abstraction is used to manage the backend connections to nodes so that + SoSNode() objects can be leveraged generically to connect to nodes, inspect + those nodes, and run commands on them. + """ + + name = 'undefined' + + def __init__(self, address, commons): + self.address = address + self.opts = commons['cmdlineopts'] + self.tmpdir = commons['tmpdir'] + self.need_sudo = commons['need_sudo'] + self._hostname = None + self.soslog = logging.getLogger('sos') + self.ui_log = logging.getLogger('sos_ui') + + def _sanitize_log_msg(self, msg): + """Attempts to obfuscate sensitive information in log messages such as + passwords""" + reg = r'(?P(pass|key|secret|PASS|KEY|SECRET).*?=)(?P.*?\s)' + return re.sub(reg, r'\g****** ', msg) + + def log_info(self, msg): + """Used to print and log info messages""" + caller = inspect.stack()[1][3] + lmsg = '[%s:%s] %s' % (self.hostname, caller, msg) + self.soslog.info(lmsg) + + def log_error(self, msg): + """Used to print and log error messages""" + caller = inspect.stack()[1][3] + lmsg = '[%s:%s] %s' % (self.hostname, caller, msg) + self.soslog.error(lmsg) + + def log_debug(self, msg): + """Used to print and log debug messages""" + msg = self._sanitize_log_msg(msg) + caller = inspect.stack()[1][3] + msg = '[%s:%s] %s' % (self.hostname, caller, msg) + self.soslog.debug(msg) + + @property + def hostname(self): + if self._hostname and 'localhost' not in self._hostname: + return self._hostname + return self.address + + @property + def connected(self): + """Is the transport __currently__ connected to the node, or otherwise + capable of seamlessly running a command or similar on the node? + """ + return False + + @property + def remote_exec(self): + """This is the command string needed to leverage the remote transport + when executing commands. For example, for an SSH transport this would + be the `ssh ` string prepended to any command so that the + command is executed by the ssh binary. + + This is also referenced by the `remote_exec` parameter for policies + when loading a policy for a remote node + """ + return None + + @classmethod + def display_help(cls, section): + if cls is RemoteTransport: + return cls.display_self_help(section) + section.set_title("%s Transport Detailed Help" + % cls.name.title().replace('_', ' ')) + if cls.__doc__ and cls.__doc__ is not RemoteTransport.__doc__: + section.add_text(cls.__doc__) + else: + section.add_text( + 'Detailed information not available for this transport' + ) + + @classmethod + def display_self_help(cls, section): + section.set_title('SoS Remote Transport Help') + section.add_text( + "\nTransports define how SoS connects to nodes and executes " + "commands on them for the purposes of an %s run. Generally, " + "this means transports define how commands are wrapped locally " + "so that they are executed on the remote node(s) instead." + % bold('sos collect') + ) + + section.add_text( + "Transports are generally selected by the cluster profile loaded " + "for a given execution, however users may explicitly set one " + "using '%s'. Note that not all transports will function for all " + "cluster/node types." + % bold('--transport=$transport_name') + ) + + section.add_text( + 'By default, OpenSSH Control Persist is attempted. Additional ' + 'information for each supported transport is available in the ' + 'following help sections:\n' + ) + + from sos.collector.sosnode import TRANSPORTS + for transport in TRANSPORTS: + _sec = bold("collect.transports.%s" % transport) + _desc = "The '%s' transport" % transport.lower() + section.add_text( + "{:>8}{:<45}{:<30}".format(' ', _sec, _desc), + newline=False + ) + + def connect(self, password): + """Perform the connection steps in order to ensure that we are able to + connect to the node for all future operations. Note that this should + not provide an interactive shell at this time. + """ + if self._connect(password): + if not self._hostname: + self._get_hostname() + return True + return False + + def _connect(self, password): + """Actually perform the connection requirements. Should be overridden + by specific transports that subclass RemoteTransport + """ + raise NotImplementedError("Transport %s does not define connect" + % self.name) + + def reconnect(self, password): + """Attempts to reconnect to the node using the standard connect() + but does not do so indefinitely. This imposes a strict number of retry + attempts before failing out + """ + attempts = 1 + last_err = 'unknown' + while attempts < 5: + self.log_debug("Attempting reconnect (#%s) to node" % attempts) + try: + if self.connect(password): + return True + except Exception as err: + self.log_debug("Attempt #%s exception: %s" % (attempts, err)) + last_err = err + attempts += 1 + self.log_error("Unable to reconnect to node after 5 attempts, " + "aborting.") + raise ConnectionException("last exception from transport: %s" + % last_err) + + def disconnect(self): + """Perform whatever steps are necessary, if any, to terminate any + connection to the node + """ + try: + if self._disconnect(): + self.log_debug("Successfully disconnected from node") + else: + self.log_error("Unable to successfully disconnect, see log for" + " more details") + except Exception as err: + self.log_error("Failed to disconnect: %s" % err) + + def _disconnect(self): + raise NotImplementedError("Transport %s does not define disconnect" + % self.name) + + def run_command(self, cmd, timeout=180, need_root=False, env=None, + get_pty=False): + """Run a command on the node, returning its output and exit code. + This should return the exit code of the command being executed, not the + exit code of whatever mechanism the transport uses to execute that + command + + :param cmd: The command to run + :type cmd: ``str`` + + :param timeout: The maximum time in seconds to allow the cmd to run + :type timeout: ``int`` + + :param get_pty: Does ``cmd`` require a pty? + :type get_pty: ``bool`` + + :param need_root: Does ``cmd`` require root privileges? + :type neeed_root: ``bool`` + + :param env: Specify env vars to be passed to the ``cmd`` + :type env: ``dict`` + + :param get_pty: Does ``cmd`` require execution with a pty? + :type get_pty: ``bool`` + + :returns: Output of ``cmd`` and the exit code + :rtype: ``dict`` with keys ``output`` and ``status`` + """ + self.log_debug('Running command %s' % cmd) + if get_pty: + cmd = "/bin/bash -c %s" % quote(cmd) + # currently we only use/support the use of pexpect for handling the + # execution of these commands, as opposed to directly invoking + # subprocess.Popen() in conjunction with tools like sshpass. + # If that changes in the future, we'll add decision making logic here + # to route to the appropriate handler, but for now we just go straight + # to using pexpect + return self._run_command_with_pexpect(cmd, timeout, need_root, env) + + def _format_cmd_for_exec(self, cmd): + """Format the command in the way needed for the remote transport to + successfully execute it as one would when manually executing it + + :param cmd: The command being executed, as formatted by SoSNode + :type cmd: ``str`` + + + :returns: The command further formatted as needed by this + transport + :rtype: ``str`` + """ + cmd = "%s %s" % (self.remote_exec, quote(cmd)) + cmd = cmd.lstrip() + return cmd + + def _run_command_with_pexpect(self, cmd, timeout, need_root, env): + """Execute the command using pexpect, which allows us to more easily + handle prompts and timeouts compared to directly leveraging the + subprocess.Popen() method. + + :param cmd: The command to execute. This will be automatically + formatted to use the transport. + :type cmd: ``str`` + + :param timeout: The maximum time in seconds to run ``cmd`` + :type timeout: ``int`` + + :param need_root: Does ``cmd`` need to run as root or with sudo? + :type need_root: ``bool`` + + :param env: Any env vars that ``cmd`` should be run with + :type env: ``dict`` + """ + cmd = self._format_cmd_for_exec(cmd) + + # if for any reason env is empty, set it to None as otherwise + # pexpect interprets this to mean "run this command with no env vars of + # any kind" + if not env: + env = None + + try: + result = pexpect.spawn(cmd, encoding='utf-8', env=env) + except pexpect.exceptions.ExceptionPexpect as err: + self.log_debug(err.value) + return {'status': 127, 'output': ''} + + _expects = [pexpect.EOF, pexpect.TIMEOUT] + if need_root and self.opts.ssh_user != 'root': + _expects.extend([ + '\\[sudo\\] password for .*:', + 'Password:' + ]) + + index = result.expect(_expects, timeout=timeout) + + if index in [2, 3]: + self._send_pexpect_password(index, result) + index = result.expect(_expects, timeout=timeout) + + if index == 0: + out = result.before + result.close() + return {'status': result.exitstatus, 'output': out} + elif index == 1: + raise CommandTimeoutException(cmd) + + def _send_pexpect_password(self, index, result): + """Handle password prompts for sudo and su usage for non-root SSH users + + :param index: The index pexpect.spawn returned to match against + either a sudo or su prompt + :type index: ``int`` + + :param result: The spawn running the command + :type result: ``pexpect.spawn`` + """ + if index == 2: + if not self.opts.sudo_pw and not self.opt.nopasswd_sudo: + msg = ("Unable to run command: sudo password " + "required but not provided") + self.log_error(msg) + raise Exception(msg) + result.sendline(self.opts.sudo_pw) + elif index == 3: + if not self.opts.root_password: + msg = ("Unable to run command as root: no root password given") + self.log_error(msg) + raise Exception(msg) + result.sendline(self.opts.root_password) + + def _get_hostname(self): + """Determine the hostname of the node and set that for future reference + and logging + + :returns: The hostname of the system, per the `hostname` command + :rtype: ``str`` + """ + _out = self.run_command('hostname') + if _out['status'] == 0: + self._hostname = _out['output'].strip() + + if not self._hostname: + self._hostname = self.address + self.log_info("Hostname set to %s" % self._hostname) + return self._hostname + + def retrieve_file(self, fname, dest): + """Copy a remote file, fname, to dest on the local node + + :param fname: The name of the file to retrieve + :type fname: ``str`` + + :param dest: Where to save the file to locally + :type dest: ``str`` + + :returns: True if file was successfully copied from remote, or False + :rtype: ``bool`` + """ + attempts = 0 + try: + while attempts < 5: + attempts += 1 + ret = self._retrieve_file(fname, dest) + if ret: + return True + self.log_info("File retrieval attempt %s failed" % attempts) + self.log_info("File retrieval failed after 5 attempts") + return False + except Exception as err: + self.log_error("Exception encountered during retrieval attempt %s " + "for %s: %s" % (attempts, fname, err)) + raise err + + def _retrieve_file(self, fname, dest): + raise NotImplementedError("Transport %s does not support file copying" + % self.name) + + def read_file(self, fname): + """Read the given file fname and return its contents + + :param fname: The name of the file to read + :type fname: ``str`` + + :returns: The content of the file + :rtype: ``str`` + """ + self.log_debug("Reading file %s" % fname) + return self._read_file(fname) + + def _read_file(self, fname): + res = self.run_command("cat %s" % fname, timeout=10) + if res['status'] == 0: + return res['output'] + else: + if 'No such file' in res['output']: + self.log_debug("File %s does not exist on node" + % fname) + else: + self.log_error("Error reading %s: %s" % + (fname, res['output'].split(':')[1:])) + return '' + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/transports/local.py sosreport-4.3/sos/collector/transports/local.py --- sosreport-4.2/sos/collector/transports/local.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/collector/transports/local.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,51 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os +import shutil + +from sos.collector.transports import RemoteTransport + + +class LocalTransport(RemoteTransport): + """ + A 'transport' to represent a local node. No remote connection is actually + made, and all commands set to be run by this transport are executed locally + without any wrappers. + """ + + name = 'local_node' + + def _connect(self, password): + return True + + def _disconnect(self): + return True + + @property + def connected(self): + return True + + def _retrieve_file(self, fname, dest): + self.log_debug("Moving %s to %s" % (fname, dest)) + shutil.copy(fname, dest) + return True + + def _format_cmd_for_exec(self, cmd): + return cmd + + def _read_file(self, fname): + if os.path.exists(fname): + with open(fname, 'r') as rfile: + return rfile.read() + self.log_debug("No such file: %s" % fname) + return '' + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/collector/transports/oc.py sosreport-4.3/sos/collector/transports/oc.py --- sosreport-4.2/sos/collector/transports/oc.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/collector/transports/oc.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,235 @@ +# Copyright Red Hat 2021, Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import json +import tempfile +import os + +from sos.collector.transports import RemoteTransport +from sos.utilities import (is_executable, sos_get_command_output, + SoSTimeoutError) + + +class OCTransport(RemoteTransport): + """ + This transport leverages the execution of commands via a locally + available and configured ``oc`` binary for OCPv4 environments. + + The location of the oc binary MUST be in the $PATH used by the locally + loaded SoS policy. Specifically this means that the binary cannot be in the + running user's home directory, such as ~/.local/bin. + + OCPv4 clusters generally discourage the use of SSH, so this transport may + be used to remove our use of SSH in favor of the environment provided + method of connecting to nodes and executing commands via debug pods. + + The debug pod created will be a privileged pod that mounts the host's + filesystem internally so that sos report collections reflect the host, and + not the container in which it runs. + + This transport will execute within a temporary 'sos-collect-tmp' project + created by the OCP cluster profile. The project will be removed at the end + of execution. + + In the event of failures due to a misbehaving OCP API or oc binary, it is + recommended to fallback to the control_persist transport by manually + setting the --transport option. + """ + + name = 'oc' + project = 'sos-collect-tmp' + + def run_oc(self, cmd, **kwargs): + """Format and run a command with `oc` in the project defined for our + execution + """ + return sos_get_command_output( + "oc -n %s %s" % (self.project, cmd), + **kwargs + ) + + @property + def connected(self): + up = self.run_oc( + "wait --timeout=0s --for=condition=ready pod/%s" % self.pod_name + ) + return up['status'] == 0 + + def get_node_pod_config(self): + """Based on our template for the debug container, add the node-specific + items so that we can deploy one of these on each node we're collecting + from + """ + return { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "%s-sos-collector" % self.address.split('.')[0], + "namespace": self.project + }, + "priorityClassName": "system-cluster-critical", + "spec": { + "volumes": [ + { + "name": "host", + "hostPath": { + "path": "/", + "type": "Directory" + } + }, + { + "name": "run", + "hostPath": { + "path": "/run", + "type": "Directory" + } + }, + { + "name": "varlog", + "hostPath": { + "path": "/var/log", + "type": "Directory" + } + }, + { + "name": "machine-id", + "hostPath": { + "path": "/etc/machine-id", + "type": "File" + } + } + ], + "containers": [ + { + "name": "sos-collector-tmp", + "image": "registry.redhat.io/rhel8/support-tools", + "command": [ + "/bin/bash" + ], + "env": [ + { + "name": "HOST", + "value": "/host" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "host", + "mountPath": "/host" + }, + { + "name": "run", + "mountPath": "/run" + }, + { + "name": "varlog", + "mountPath": "/var/log" + }, + { + "name": "machine-id", + "mountPath": "/etc/machine-id" + } + ], + "securityContext": { + "privileged": True, + "runAsUser": 0 + }, + "stdin": True, + "stdinOnce": True, + "tty": True + } + ], + "restartPolicy": "Never", + "nodeName": self.address, + "hostNetwork": True, + "hostPID": True, + "hostIPC": True + } + } + + def _connect(self, password): + # the oc binary must be _locally_ available for this to work + if not is_executable('oc'): + return False + + # deploy the debug container we'll exec into + podconf = self.get_node_pod_config() + self.pod_name = podconf['metadata']['name'] + fd, self.pod_tmp_conf = tempfile.mkstemp(dir=self.tmpdir) + with open(fd, 'w') as cfile: + json.dump(podconf, cfile) + self.log_debug("Starting sos collector container '%s'" % self.pod_name) + # this specifically does not need to run with a project definition + out = sos_get_command_output( + "oc create -f %s" % self.pod_tmp_conf + ) + if (out['status'] != 0 or "pod/%s created" % self.pod_name not in + out['output']): + self.log_error("Unable to deploy sos collect pod") + self.log_debug("Debug pod deployment failed: %s" % out['output']) + return False + self.log_debug("Pod '%s' successfully deployed, waiting for pod to " + "enter ready state" % self.pod_name) + + # wait for the pod to report as running + try: + up = self.run_oc("wait --for=condition=Ready pod/%s --timeout=30s" + % self.pod_name, + # timeout is for local safety, not oc + timeout=40) + if not up['status'] == 0: + self.log_error("Pod not available after 30 seconds") + return False + except SoSTimeoutError: + self.log_error("Timeout while polling for pod readiness") + return False + except Exception as err: + self.log_error("Error while waiting for pod to be ready: %s" + % err) + return False + + return True + + def _format_cmd_for_exec(self, cmd): + if cmd.startswith('oc'): + return ("oc -n %s exec --request-timeout=0 %s -- chroot /host %s" + % (self.project, self.pod_name, cmd)) + return super(OCTransport, self)._format_cmd_for_exec(cmd) + + def run_command(self, cmd, timeout=180, need_root=False, env=None, + get_pty=False): + # debug pod setup is slow, extend all timeouts to account for this + if timeout: + timeout += 10 + + # since we always execute within a bash shell, force disable get_pty + # to avoid double-quoting + return super(OCTransport, self).run_command(cmd, timeout, need_root, + env, False) + + def _disconnect(self): + if os.path.exists(self.pod_tmp_conf): + os.unlink(self.pod_tmp_conf) + removed = self.run_oc("delete pod %s" % self.pod_name) + if "deleted" not in removed['output']: + self.log_debug("Calling delete on pod '%s' failed: %s" + % (self.pod_name, removed)) + return False + return True + + @property + def remote_exec(self): + return ("oc -n %s exec --request-timeout=0 %s -- /bin/bash -c" + % (self.project, self.pod_name)) + + def _retrieve_file(self, fname, dest): + cmd = self.run_oc("cp %s:%s %s" % (self.pod_name, fname, dest)) + return cmd['status'] == 0 diff -Nru sosreport-4.2/sos/component.py sosreport-4.3/sos/component.py --- sosreport-4.2/sos/component.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/component.py 2022-02-15 04:20:20.000000000 +0000 @@ -14,6 +14,7 @@ import os import tempfile import sys +import time from argparse import SUPPRESS from datetime import datetime @@ -22,7 +23,7 @@ from sos import __version__ from sos.archive import TarFileArchive from sos.options import SoSOptions -from sos.utilities import TempFileUtil +from sos.utilities import TempFileUtil, shell_out class SoSComponent(): @@ -49,6 +50,7 @@ arg_defaults = {} configure_logging = True load_policy = True + load_probe = True root_required = False _arg_defaults = { @@ -105,13 +107,7 @@ self._setup_logging() if self.load_policy: - try: - import sos.policies - self.policy = sos.policies.load(sysroot=self.opts.sysroot) - self.sysroot = self.policy.host_sysroot() - except KeyboardInterrupt: - self._exit(0) - self._is_root = self.policy.is_root() + self.load_local_policy() if self.manifest is not None: self.manifest.add_field('version', __version__) @@ -121,9 +117,21 @@ self.manifest.add_field('end_time', '') self.manifest.add_field('run_time', '') self.manifest.add_field('compression', '') + self.manifest.add_field('tmpdir', self.tmpdir) + self.manifest.add_field('tmpdir_fs_type', self.tmpfstype) self.manifest.add_field('policy', self.policy.distro) self.manifest.add_section('components') + def load_local_policy(self): + try: + import sos.policies + self.policy = sos.policies.load(sysroot=self.opts.sysroot, + probe_runtime=self.load_probe) + self.sysroot = self.policy.sysroot + except KeyboardInterrupt: + self._exit(0) + self._is_root = self.policy.is_root() + def execute(self): raise NotImplementedError @@ -133,7 +141,10 @@ self._exit() return exit_handler - def _exit(self, error=0): + def _exit(self, error=0, msg=None): + if msg: + self.ui_log.error("") + self.ui_log.error(msg) raise SystemExit(error) def get_tmpdir_default(self): @@ -142,13 +153,27 @@ use a standardized env var to redirect to the host's filesystem instead """ if self.opts.tmp_dir: - return os.path.abspath(self.opts.tmp_dir) - - tmpdir = '/var/tmp' + tmpdir = os.path.abspath(self.opts.tmp_dir) + else: + tmpdir = os.getenv('TMPDIR', None) or '/var/tmp' if os.getenv('HOST', None) and os.getenv('container', None): tmpdir = os.path.join(os.getenv('HOST'), tmpdir.lstrip('/')) + # no standard library method exists for this, so call out to stat to + # avoid bringing in a dependency on psutil + self.tmpfstype = shell_out( + "stat --file-system --format=%s %s" % ("%T", tmpdir) + ).strip() + + if self.tmpfstype == 'tmpfs': + # can't log to the ui or sos.log yet since those require a defined + # tmpdir to setup + print("WARNING: tmp-dir is set to a tmpfs filesystem. This may " + "increase memory pressure and cause instability on low " + "memory systems, or when using --all-logs.") + time.sleep(2) + return tmpdir def check_listing_options(self): diff -Nru sosreport-4.2/sos/help/__init__.py sosreport-4.3/sos/help/__init__.py --- sosreport-4.2/sos/help/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/help/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,302 @@ +# Copyright 2020 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import inspect +import importlib +import sys +import os + +from collections import OrderedDict +from sos.component import SoSComponent +from sos.policies import import_policy +from sos.report.plugins import Plugin +from sos.utilities import bold, ImporterHelper +from textwrap import fill + +try: + TERMSIZE = min(os.get_terminal_size().columns, 120) +except Exception: + TERMSIZE = 120 + + +class SoSHelper(SoSComponent): + """Provide better, more in-depth help for specific parts of sos than is + provided in either standard --help output or in manpages. + """ + + desc = 'Detailed help infomation' + configure_logging = False + load_policy = False + load_probe = False + + arg_defaults = { + 'topic': '' + } + + def __init__(self, parser, args, cmdline): + super(SoSHelper, self).__init__(parser, args, cmdline) + self.topic = self.opts.topic + + @classmethod + def add_parser_options(cls, parser): + parser.usage = 'sos help TOPIC [options]' + help_grp = parser.add_argument_group( + 'Help Information Options', + 'These options control what detailed information is displayed' + ) + help_grp.add_argument('topic', metavar='TOPIC', default='', nargs='?', + help=('name of the topic or component to show ' + 'help for')) + + def sanitize_topic_component(self): + _com = self.opts.topic.split('.')[0] + _replace = { + 'clean': 'cleaner', + 'mask': 'cleaner', + 'collect': 'collector' + } + if _com in _replace: + self.opts.topic = self.opts.topic.replace(_com, _replace[_com]) + + def execute(self): + if not self.opts.topic: + self.display_self_help() + sys.exit(0) + + # standardize the command to the module naming pattern + self.sanitize_topic_component() + + try: + klass = self.get_obj_for_topic() + except Exception as err: + print("Could not load help for '%s': %s" % (self.opts.topic, err)) + sys.exit(1) + + if klass: + try: + ht = HelpSection() + klass.display_help(ht) + ht.display() + except Exception as err: + print("Error loading help: %s" % err) + else: + print("No help section found for '%s'" % self.opts.topic) + + def get_obj_for_topic(self): + """Based on the help topic we're after, try to smartly decide which + object we need to manipulate in order to get help information. + """ + static_map = { + 'report': 'SoSReport', + 'report.plugins': 'Plugin', + 'cleaner': 'SoSCleaner', + 'collector': 'SoSCollector', + 'collector.transports': 'RemoteTransport', + 'collector.clusters': 'Cluster', + 'policies': 'Policy' + } + + cls = None + + if self.opts.topic in static_map: + mod = importlib.import_module('sos.' + self.opts.topic) + cls = getattr(mod, static_map[self.opts.topic]) + else: + _help = { + 'report.plugins.': self._get_plugin_variant, + 'policies.': self._get_policy_by_name, + 'collector.transports.': self._get_collect_transport, + 'collector.clusters.': self._get_collect_cluster, + } + for _sec in _help: + if self.opts.topic.startswith(_sec): + cls = _help[_sec]() + break + return cls + + def _get_collect_transport(self): + from sos.collector.sosnode import TRANSPORTS + _transport = self.opts.topic.split('.')[-1] + if _transport in TRANSPORTS: + return TRANSPORTS[_transport] + + def _get_collect_cluster(self): + from sos.collector import SoSCollector + import sos.collector.clusters + clusters = SoSCollector._load_modules(sos.collector.clusters, + 'clusters') + for cluster in clusters: + if cluster[0] == self.opts.topic.split('.')[-1]: + return cluster[1] + + def _get_plugin_variant(self): + mod = importlib.import_module('sos.' + self.opts.topic) + self.load_local_policy() + mems = inspect.getmembers(mod, inspect.isclass) + plugins = [m[1] for m in mems if issubclass(m[1], Plugin)] + for plugin in plugins: + if plugin.__subclasses__(): + cls = self.policy.match_plugin(plugin.__subclasses__()) + return cls + + def _get_policy_by_name(self): + _topic = self.opts.topic.split('.')[-1] + # mimic policy loading to discover all policiy classes without + # needing to manually define each here + import sos.policies.distros + _helper = ImporterHelper(sos.policies.distros) + for mod in _helper.get_modules(): + for policy in import_policy(mod): + _p = policy.__name__.lower().replace('policy', '') + if _p == _topic: + return policy + + def display_self_help(self): + """Displays the help information for this component directly, that is + help for `sos help`. + """ + self_help = HelpSection( + 'Detailed help for sos help', + ('The \'help\' sub-command is used to provide more detailed ' + 'information on different sub-commands available to sos as well ' + 'as different components at play within those sub-commands.') + ) + self_help.add_text( + 'SoS - officially pronounced "ess-oh-ess" - is a diagnostic and ' + 'supportability utility used by several Linux distributions as an ' + 'easy-to-use tool for standardized data collection. The most known' + ' component of which is %s (formerly sosreport) which is used to ' + 'collect troubleshooting information into an archive for review ' + 'by sysadmins or technical support teams.' + % bold('sos report') + ) + + subsect = self_help.add_section('How to search using sos help') + usage = bold('$component.$topic.$subtopic') + subsect.add_text( + 'To get more information on a given topic, use the form \'%s\'.' + % usage + ) + + rep_ex = bold('sos help report.plugins.kernel') + subsect.add_text("For example '%s' will provide more information on " + "the kernel plugin for the report function." % rep_ex) + + avail_help = self_help.add_section('Available Help Sections') + avail_help.add_text( + 'The following help sections are available. Additional help' + ' topics and subtopics may be displayed within their respective ' + 'help section.\n' + ) + + sections = { + 'report': 'Detailed help on the report command', + 'report.plugins': 'Information on the plugin design of sos', + 'report.plugins.$plugin': 'Information on a specific $plugin', + 'clean': 'Detailed help on the clean command', + 'collect': 'Detailed help on the collect command', + 'policies': 'How sos operates on different distributions' + } + + for sect in sections: + avail_help.add_text( + "\t{:<36}{}".format(bold(sect), sections[sect]), + newline=False + ) + + self_help.display() + + +class HelpSection(): + """This class is used to build the output displayed by `sos help` in a + standard fashion that provides easy formatting controls. + """ + + def __init__(self, title='', content='', indent=''): + """ + :param title: The title of the output section, will be prominently + displayed + :type title: ``str`` + + :param content: The text content to be displayed with this section + :type content: ``str`` + + :param indent: If the section should be nested, set this to a multiple + of 4. + :type indent: ``int`` + """ + self.title = title + self.content = content + self.indent = indent + self.sections = OrderedDict() + + def set_title(self, title): + """Set or override the title for this help section + + :param title: The name to set for this help section + :type title: ``str`` + """ + self.title = title + + def add_text(self, content, newline=True): + """Add body text to this section. If content for this section already + exists, append the new ``content`` after a newline. + + :param content: The text to add to the section + :type content: ``str`` + """ + if self.content: + ln = '\n\n' if newline else '\n' + content = ln + content + self.content += content + + def add_section(self, title, content='', indent=''): + """Add a section of text to the help section that will be displayed + when the HelpSection object is printed. + + Sections will be printed *in the order added*. + + This will return a subsection object with which block(s) of text may be + added to the subsection associated with ``title``. + + :param title: The title of the subsection being added + :type title: ``str`` + + :param content: The text the new section should contain + :type content: ``str`` + + :returns: The newly created subsection for ``title`` + :rtype: ``HelpSection`` + """ + self._add_section(title, content, indent) + return self.sections[title] + + def _add_section(self, title, content='', indent=''): + """Internal method used to add a new subsection to this help output + + :param title: The title of the subsection being added + :type title: ``str` + """ + if title in self.sections: + raise Exception('A section with that title already exists') + self.sections[title] = HelpSection(title, content, indent) + + def display(self): + """Print the HelpSection contents, including any subsections, to + console. + """ + print(fill( + bold(self.title), width=TERMSIZE, initial_indent=self.indent + )) + for ln in self.content.splitlines(): + print(fill(ln, width=TERMSIZE, initial_indent=self.indent)) + for section in self.sections: + print('') + self.sections[section].display() diff -Nru sosreport-4.2/sos/__init__.py sosreport-4.3/sos/__init__.py --- sosreport-4.2/sos/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -14,7 +14,7 @@ This module houses the i18n setup and message function. The default is to use gettext to internationalize messages. """ -__version__ = "4.2" +__version__ = "4.3" import os import sys @@ -59,9 +59,11 @@ # if no aliases are desired, pass an empty list import sos.report import sos.cleaner + import sos.help self._components = { 'report': (sos.report.SoSReport, ['rep']), - 'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']) + 'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']), + 'help': (sos.help.SoSHelper, []) } # some distros do not want pexpect as a default dep, so try to load # collector here, and if it fails add an entry that implies it is at diff -Nru sosreport-4.2/sos/options.py sosreport-4.3/sos/options.py --- sosreport-4.2/sos/options.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/options.py 2022-02-15 04:20:20.000000000 +0000 @@ -200,7 +200,10 @@ odict[rename_opts[key]] = odict.pop(key) # set the values according to the config file for key, val in odict.items(): - if isinstance(val, str): + # most option values do not tolerate spaces, special + # exception however for --keywords which we do want to + # support phrases, and thus spaces, for + if isinstance(val, str) and key != 'keywords': val = val.replace(' ', '') if key not in self.arg_defaults: # read an option that is not loaded by the current @@ -281,6 +284,8 @@ null_values = ("False", "None", "[]", '""', "''", "0") if not value or value in null_values: return False + if name == 'plugopts' and value: + return True if name in self.arg_defaults: if str(value) == str(self.arg_defaults[name]): return False diff -Nru sosreport-4.2/sos/policies/distros/debian.py sosreport-4.3/sos/policies/distros/debian.py --- sosreport-4.2/sos/policies/distros/debian.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/distros/debian.py 2022-02-15 04:20:20.000000000 +0000 @@ -26,8 +26,9 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True, remote_exec=None): super(DebianPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) - self.package_manager = DpkgPackageManager(chroot=sysroot, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + self.package_manager = DpkgPackageManager(chroot=self.sysroot, remote_exec=remote_exec) self.valid_subclasses += [DebianPlugin] diff -Nru sosreport-4.2/sos/policies/distros/__init__.py sosreport-4.3/sos/policies/distros/__init__.py --- sosreport-4.2/sos/policies/distros/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/distros/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -17,10 +17,11 @@ from sos.policies import Policy from sos.policies.init_systems import InitSystem from sos.policies.init_systems.systemd import SystemdInit +from sos.policies.runtimes.crio import CrioContainerRuntime from sos.policies.runtimes.podman import PodmanContainerRuntime from sos.policies.runtimes.docker import DockerContainerRuntime -from sos.utilities import shell_out, is_executable +from sos.utilities import shell_out, is_executable, bold try: @@ -29,6 +30,10 @@ except ImportError: REQUESTS_LOADED = False +# Container environment variables for detecting if we're in a container +ENV_CONTAINER = 'container' +ENV_HOST_SYSROOT = 'HOST' + class LinuxPolicy(Policy): """This policy is meant to be an abc class that provides common @@ -64,15 +69,23 @@ container_version_command = None container_authfile = None - def __init__(self, sysroot=None, init=None, probe_runtime=True): + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): super(LinuxPolicy, self).__init__(sysroot=sysroot, - probe_runtime=probe_runtime) + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + if sysroot: + self.sysroot = sysroot + else: + self.sysroot = self._container_init() or '/' + self.init_kernel_modules() if init is not None: self.init_system = init elif os.path.isdir("/run/systemd/system/"): - self.init_system = SystemdInit() + self.init_system = SystemdInit(chroot=self.sysroot) else: self.init_system = InitSystem() @@ -80,7 +93,8 @@ if self.probe_runtime: _crun = [ PodmanContainerRuntime(policy=self), - DockerContainerRuntime(policy=self) + DockerContainerRuntime(policy=self), + CrioContainerRuntime(policy=self) ] for runtime in _crun: if runtime.check_is_active(): @@ -130,20 +144,105 @@ def sanitize_filename(self, name): return re.sub(r"[^-a-z,A-Z.0-9]", "", name) + @classmethod + def display_help(cls, section): + if cls == LinuxPolicy: + cls.display_self_help(section) + else: + section.set_title("%s Distribution Policy" % cls.distro) + cls.display_distro_help(section) + + @classmethod + def display_self_help(cls, section): + section.set_title("SoS Distribution Policies") + section.add_text( + 'Distributions supported by SoS will each have a specific policy ' + 'defined for them, to ensure proper operation of SoS on those ' + 'systems.' + ) + + @classmethod + def display_distro_help(cls, section): + if cls.__doc__ and cls.__doc__ is not LinuxPolicy.__doc__: + section.add_text(cls.__doc__) + else: + section.add_text( + '\nDetailed help information for this policy is not available' + ) + + # instantiate the requested policy so we can report more interesting + # information like $PATH and loaded presets + _pol = cls(None, None, False) + section.add_text( + "Default --upload location: %s" % _pol._upload_url + ) + section.add_text( + "Default container runtime: %s" % _pol.default_container_runtime, + newline=False + ) + section.add_text( + "$PATH used when running report: %s" % _pol.PATH, + newline=False + ) + + refsec = section.add_section('Reference URLs') + for url in cls.vendor_urls: + refsec.add_text( + "{:>8}{:<30}{:<40}".format(' ', url[0], url[1]), + newline=False + ) + + presec = section.add_section('Presets Available With This Policy\n') + presec.add_text( + bold( + "{:>8}{:<20}{:<45}{:<30}".format(' ', 'Preset Name', + 'Description', + 'Enabled Options') + ), + newline=False + ) + for preset in _pol.presets: + _preset = _pol.presets[preset] + _opts = ' '.join(_preset.opts.to_args()) + presec.add_text( + "{:>8}{:<20}{:<45}{:<30}".format( + ' ', preset, _preset.desc, _opts + ), + newline=False + ) + + def _container_init(self): + """Check if sos is running in a container and perform container + specific initialisation based on ENV_HOST_SYSROOT. + """ + if ENV_CONTAINER in os.environ: + if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']: + self._in_container = True + if ENV_HOST_SYSROOT in os.environ: + _host_sysroot = os.environ[ENV_HOST_SYSROOT] + use_sysroot = self._in_container and _host_sysroot is not None + if use_sysroot: + host_tmp_dir = os.path.abspath(_host_sysroot + self._tmp_dir) + self._tmp_dir = host_tmp_dir + return _host_sysroot if use_sysroot else None + def init_kernel_modules(self): """Obtain a list of loaded kernel modules to reference later for plugin enablement and SoSPredicate checks """ self.kernel_mods = [] + release = os.uname().release # first load modules from lsmod - lines = shell_out("lsmod", timeout=0).splitlines() + lines = shell_out("lsmod", timeout=0, chroot=self.sysroot).splitlines() self.kernel_mods.extend([ line.split()[0].strip() for line in lines[1:] ]) # next, include kernel builtins - builtins = "/usr/lib/modules/%s/modules.builtin" % os.uname().release + builtins = self.join_sysroot( + "/usr/lib/modules/%s/modules.builtin" % release + ) try: with open(builtins, "r") as mfile: for line in mfile: @@ -160,7 +259,7 @@ 'dm_mod': 'CONFIG_BLK_DEV_DM' } - booted_config = "/boot/config-%s" % os.uname().release + booted_config = self.join_sysroot("/boot/config-%s" % release) kconfigs = [] try: with open(booted_config, "r") as kfile: @@ -174,6 +273,11 @@ if config_strings[builtin] in kconfigs: self.kernel_mods.append(builtin) + def join_sysroot(self, path): + if self.sysroot and self.sysroot != '/': + path = os.path.join(self.sysroot, path.lstrip('/')) + return path + def pre_work(self): # this method will be called before the gathering begins @@ -193,11 +297,12 @@ cmdline_opts.quiet: try: if caseid: - self.case_id = caseid + self.commons['cmdlineopts'].case_id = caseid else: - self.case_id = input(_("Optionally, please enter the case " - "id that you are generating this " - "report for [%s]: ") % caseid) + self.commons['cmdlineopts'].case_id = input( + _("Optionally, please enter the case id that you are " + "generating this report for [%s]: ") % caseid + ) # Policies will need to handle the prompts for user information if cmdline_opts.upload and self.get_upload_url(): self.prompt_for_upload_user() @@ -446,7 +551,8 @@ put_expects = [ u'100%', pexpect.TIMEOUT, - pexpect.EOF + pexpect.EOF, + u'No such file or directory' ] put_success = ret.expect(put_expects, timeout=180) @@ -458,6 +564,8 @@ raise Exception("Timeout expired while uploading") elif put_success == 2: raise Exception("Unknown error during upload: %s" % ret.before) + elif put_success == 3: + raise Exception("Unable to write archive to destination") else: raise Exception("Unexpected response from server: %s" % ret.before) @@ -524,7 +632,7 @@ r = self._upload_https_put(arc, verify) else: r = self._upload_https_post(arc, verify) - if r.status_code != 201: + if r.status_code != 200 and r.status_code != 201: if r.status_code == 401: raise Exception( "Authentication failed: invalid user credentials" diff -Nru sosreport-4.2/sos/policies/distros/openeuler.py sosreport-4.3/sos/policies/distros/openeuler.py --- sosreport-4.2/sos/policies/distros/openeuler.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/policies/distros/openeuler.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,43 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import OpenEulerPlugin +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class OpenEulerPolicy(RedHatPolicy): + distro = "openEuler" + vendor = "The openEuler Project" + vendor_urls = [('Distribution Website', 'https://openeuler.org/')] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(OpenEulerPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + self.valid_subclasses += [OpenEulerPlugin] + + @classmethod + def check(cls, remote=''): + + if remote: + return cls.distro in remote + + if not os.path.exists(OS_RELEASE): + return False + + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'openEuler' in line: + return True + return False + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/policies/distros/redhat.py sosreport-4.3/sos/policies/distros/redhat.py --- sosreport-4.2/sos/policies/distros/redhat.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/distros/redhat.py 2022-02-15 04:20:20.000000000 +0000 @@ -17,8 +17,9 @@ from sos.presets.redhat import (RHEL_PRESETS, ATOMIC_PRESETS, RHV, RHEL, CB, RHOSP, RHOCP, RH_CFME, RH_SATELLITE, ATOMIC) -from sos.policies.distros import LinuxPolicy +from sos.policies.distros import LinuxPolicy, ENV_HOST_SYSROOT from sos.policies.package_managers.rpm import RpmPackageManager +from sos.utilities import bold from sos import _sos as _ try: @@ -42,7 +43,6 @@ _redhat_release = '/etc/redhat-release' _tmp_dir = "/var/tmp" _in_container = False - _host_sysroot = '/' default_scl_prefix = '/opt/rh' name_pattern = 'friendly' upload_url = None @@ -54,16 +54,11 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True, remote_exec=None): super(RedHatPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) + probe_runtime=probe_runtime, + remote_exec=remote_exec) self.usrmove = False - # need to set _host_sysroot before PackageManager() - if sysroot: - self._container_init() - self._host_sysroot = sysroot - else: - sysroot = self._container_init() - self.package_manager = RpmPackageManager(chroot=sysroot, + self.package_manager = RpmPackageManager(chroot=self.sysroot, remote_exec=remote_exec) self.valid_subclasses += [RedHatPlugin] @@ -83,7 +78,8 @@ self.PATH = "/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" self.PATH += os.pathsep + "/usr/local/bin" self.PATH += os.pathsep + "/usr/local/sbin" - self.set_exec_path() + if not self.remote_exec: + self.set_exec_path() self.load_presets() @classmethod @@ -98,6 +94,31 @@ """ return False + @classmethod + def display_distro_help(cls, section): + if cls is not RedHatPolicy: + super(RedHatPolicy, cls).display_distro_help(section) + return + section.add_text( + 'This policy is a building block for all other Red Hat family ' + 'distributions. You are likely looking for one of the ' + 'distributions listed below.\n' + ) + + subs = { + 'centos': CentOsPolicy, + 'rhel': RHELPolicy, + 'redhatcoreos': RedHatCoreOSPolicy, + 'fedora': FedoraPolicy + } + + for subc in subs: + subln = bold("policies.%s" % subc) + section.add_text( + "{:>8}{:<35}{:<30}".format(' ', subln, subs[subc].distro), + newline=False + ) + def check_usrmove(self, pkgs): """Test whether the running system implements UsrMove. @@ -140,21 +161,6 @@ else: return files - def _container_init(self): - """Check if sos is running in a container and perform container - specific initialisation based on ENV_HOST_SYSROOT. - """ - if ENV_CONTAINER in os.environ: - if os.environ[ENV_CONTAINER] in ['docker', 'oci', 'podman']: - self._in_container = True - if ENV_HOST_SYSROOT in os.environ: - self._host_sysroot = os.environ[ENV_HOST_SYSROOT] - use_sysroot = self._in_container and self._host_sysroot is not None - if use_sysroot: - host_tmp_dir = os.path.abspath(self._host_sysroot + self._tmp_dir) - self._tmp_dir = host_tmp_dir - return self._host_sysroot if use_sysroot else None - def runlevel_by_service(self, name): from subprocess import Popen, PIPE ret = [] @@ -183,10 +189,6 @@ return opt_tmp_dir -# Container environment variables on Red Hat systems. -ENV_CONTAINER = 'container' -ENV_HOST_SYSROOT = 'HOST' - # Legal disclaimer text for Red Hat products disclaimer_text = """ Any information provided to %(vendor)s will be treated in \ @@ -200,11 +202,37 @@ No changes will be made to system configuration. """ -RH_API_HOST = "https://access.redhat.com" +RH_API_HOST = "https://api.access.redhat.com" RH_SFTP_HOST = "sftp://sftp.access.redhat.com" class RHELPolicy(RedHatPolicy): + """ + The RHEL policy is used specifically for Red Hat Enterprise Linux, of + any release, and not forks or derivative distributions. For example, this + policy will be loaded for any RHEL 8 installation, but will not be loaded + for CentOS Stream 8 or Red Hat CoreOS, for which there are separate + policies. + + Plugins activated by installed packages will only be activated if those + packages are installed via RPM (dnf/yum inclusive). Packages installed by + other means are not considered by this policy. + + By default, --upload will be directed to using the SFTP location provided + by Red Hat for technical support cases. Users who provide login credentials + for their Red Hat Customer Portal account will have their archives uploaded + to a user-specific directory. + + If users provide those credentials as well as a case number, --upload will + instead attempt to directly upload archives to the referenced case, thus + streamlining the process of providing data to technical support engineers. + + If either or both of the credentials or case number are omitted or are + incorrect, then a temporary anonymous user will be used for upload to the + SFTP server, and users will need to provide that information to their + technical support engineer. This information will be printed at the end of + the upload process for any sos report execution. + """ distro = RHEL_RELEASE_STR vendor = "Red Hat" msg = _("""\ @@ -275,7 +303,7 @@ elif self.commons['cmdlineopts'].upload_protocol == 'sftp': return RH_SFTP_HOST else: - rh_case_api = "/hydra/rest/cases/%s/attachments" + rh_case_api = "/support/v1/cases/%s/attachments" return RH_API_HOST + rh_case_api % self.case_id def _get_upload_headers(self): @@ -294,10 +322,10 @@ """The RH SFTP server will only automatically connect file uploads to cases if the filename _starts_ with the case number """ + fname = self.upload_archive_name.split('/')[-1] if self.case_id: - return "%s_%s" % (self.case_id, - self.upload_archive_name.split('/')[-1]) - return self.upload_archive_name + return "%s_%s" % (self.case_id, fname) + return fname def upload_sftp(self): """Override the base upload_sftp to allow for setting an on-demand @@ -312,12 +340,12 @@ " for obtaining SFTP auth token.") _token = None _user = None + url = RH_API_HOST + '/support/v2/sftp/token' # we have a username and password, but we need to reset the password # to be the token returned from the auth endpoint if self.get_upload_user() and self.get_upload_password(): - url = RH_API_HOST + '/hydra/rest/v1/sftp/token' auth = self.get_upload_https_auth() - ret = requests.get(url, auth=auth, timeout=10) + ret = requests.post(url, auth=auth, timeout=10) if ret.status_code == 200: # credentials are valid _user = self.get_upload_user() @@ -327,8 +355,8 @@ "credentials. Will try anonymous.") # we either do not have a username or password/token, or both if not _token: - aurl = RH_API_HOST + '/hydra/rest/v1/sftp/token?isAnonymous=true' - anon = requests.get(aurl, timeout=10) + adata = {"isAnonymous": True} + anon = requests.post(url, data=json.dumps(adata), timeout=10) if anon.status_code == 200: resp = json.loads(anon.text) _user = resp['username'] @@ -475,6 +503,26 @@ class RedHatCoreOSPolicy(RHELPolicy): + """ + Red Hat CoreOS is a containerized host built upon Red Hat Enterprise Linux + and as such this policy is built on top of the RHEL policy. For users, this + should be entirely transparent as any behavior exhibited or influenced on + RHEL systems by that policy will be seen on RHCOS systems as well. + + The one change is that this policy ensures that sos collect will deploy a + container on RHCOS systems in order to facilitate sos report collection, + as RHCOS discourages non-default package installation via rpm-ostree which + is used to maintain atomicity for RHCOS nodes. The default container image + used by this policy is the support-tools image maintained by Red Hat on + registry.redhat.io. + + Note that this policy is only loaded when sos is directly run on an RHCOS + node - if sos collect uses the `oc` transport (the default transport that + will be attempted by the ocp cluster profile), then the policy loaded + inside the launched pod will be RHEL. Again, this is expected and will not + impact how sos report collections are performed. + """ + distro = "Red Hat CoreOS" msg = _("""\ This command will collect diagnostic and configuration \ @@ -545,6 +593,18 @@ class FedoraPolicy(RedHatPolicy): + """ + The policy for Fedora based systems, regardless of spin/edition. This + policy is based on the parent Red Hat policy, and thus will only check for + RPM packages when considering packaged-based plugin enablement. Packages + installed by other sources are not considered. + + There is no default --upload location for this policy. If users need to + upload an sos report archive from a Fedora system, they will need to + provide the location via --upload-url, and optionally login credentials + for that location via --upload-user and --upload-pass (or the appropriate + environment variables). + """ distro = "Fedora" vendor = "the Fedora Project" diff -Nru sosreport-4.2/sos/policies/distros/rocky.py sosreport-4.3/sos/policies/distros/rocky.py --- sosreport-4.2/sos/policies/distros/rocky.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/policies/distros/rocky.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,52 @@ +# Copyright (C) Louis Abel + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.distros.redhat import RedHatPolicy, OS_RELEASE +import os + + +class RockyPolicy(RedHatPolicy): + distro = "Rocky Linux" + vendor = "Rocky Enterprise Software Foundation" + vendor_urls = [ + ('Distribution Website', 'https://rockylinux.org'), + ('Vendor Website', 'https://resf.org') + ] + + def __init__(self, sysroot=None, init=None, probe_runtime=True, + remote_exec=None): + super(RockyPolicy, self).__init__(sysroot=sysroot, init=init, + probe_runtime=probe_runtime, + remote_exec=remote_exec) + + @classmethod + def check(cls, remote=''): + if remote: + return cls.distro in remote + + # Return False if /etc/os-release is missing + if not os.path.exists(OS_RELEASE): + return False + + # Return False if /etc/rocky-release is missing + if not os.path.isfile('/etc/rocky-release'): + return False + + # If we've gotten this far, check for Rocky in + # /etc/os-release + with open(OS_RELEASE, 'r') as f: + for line in f: + if line.startswith('NAME'): + if 'Rocky Linux' in line: + return True + + return False + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/policies/distros/suse.py sosreport-4.3/sos/policies/distros/suse.py --- sosreport-4.2/sos/policies/distros/suse.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/distros/suse.py 2022-02-15 04:20:20.000000000 +0000 @@ -25,7 +25,8 @@ def __init__(self, sysroot=None, init=None, probe_runtime=True, remote_exec=None): super(SuSEPolicy, self).__init__(sysroot=sysroot, init=init, - probe_runtime=probe_runtime) + probe_runtime=probe_runtime, + remote_exec=remote_exec) self.valid_subclasses += [SuSEPlugin, RedHatPlugin] self.usrmove = False diff -Nru sosreport-4.2/sos/policies/__init__.py sosreport-4.3/sos/policies/__init__.py --- sosreport-4.2/sos/policies/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -10,7 +10,8 @@ from sos.presets import (NO_PRESET, GENERIC_PRESETS, PRESETS_PATH, PresetDefaults, DESC, NOTE, OPTS) from sos.policies.package_managers import PackageManager -from sos.utilities import ImporterHelper, import_module, get_human_readable +from sos.utilities import (ImporterHelper, import_module, get_human_readable, + bold) from sos.report.plugins import IndependentPlugin, ExperimentalPlugin from sos.options import SoSOptions from sos import _sos as _ @@ -45,7 +46,7 @@ return cache['policy'] -class Policy(object): +class Policy(): """Policies represent distributions that sos supports, and define the way in which sos behaves on those distributions. A policy should define at minimum a way to identify the distribution, and a package manager to allow @@ -110,9 +111,8 @@ presets = {"": PresetDefaults()} presets_path = PRESETS_PATH _in_container = False - _host_sysroot = '/' - def __init__(self, sysroot=None, probe_runtime=True): + def __init__(self, sysroot=None, probe_runtime=True, remote_exec=None): """Subclasses that choose to override this initializer should call super() to ensure that they get the required platform bits attached. super(SubClass, self).__init__(). Policies that require runtime @@ -123,8 +123,10 @@ self.probe_runtime = probe_runtime self.package_manager = PackageManager() self.valid_subclasses = [IndependentPlugin] - self.set_exec_path() - self._host_sysroot = sysroot + self.remote_exec = remote_exec + if not self.remote_exec: + self.set_exec_path() + self.sysroot = sysroot self.register_presets(GENERIC_PRESETS) def check(self, remote=''): @@ -177,14 +179,6 @@ """ return self._in_container - def host_sysroot(self): - """Get the host's default sysroot - - :returns: Host sysroot - :rtype: ``str`` or ``None`` - """ - return self._host_sysroot - def dist_version(self): """ Return the OS version @@ -362,6 +356,49 @@ to use""" return "sha256" + @classmethod + def display_help(self, section): + section.set_title('SoS Policies') + section.add_text( + 'Policies help govern how SoS operates on across different distri' + 'butions of Linux. They control aspects such as plugin enablement,' + ' $PATH determination, how/which package managers are queried, ' + 'default upload specifications, and more.' + ) + + section.add_text( + "When SoS intializes most functions, for example %s and %s, one " + "of the first operations is to determine the correct policy to " + "load for the local system. Policies will determine the proper " + "package manager to use, any applicable container runtime(s), and " + "init systems so that SoS and report plugins can properly function" + " for collections. Generally speaking a single policy will map to" + " a single distribution; for example there are separate policies " + "for Debian, Ubuntu, RHEL, and Fedora." + % (bold('sos report'), bold('sos collect')) + ) + + section.add_text( + "It is currently not possible for users to directly control which " + "policy is loaded." + ) + + pols = { + 'policies.cos': 'The Google Cloud-Optimized OS distribution', + 'policies.debian': 'The Debian distribution', + 'policies.redhat': ('Red Hat family distributions, not necessarily' + ' including forks'), + 'policies.ubuntu': 'Ubuntu/Canonical distributions' + } + + seealso = section.add_section('See Also') + seealso.add_text( + "For more information on distribution policies, see below\n" + ) + for pol in pols: + seealso.add_text("{:>8}{:<20}{:<30}".format(' ', pol, pols[pol]), + newline=False) + def display_results(self, archive, directory, checksum, archivestat=None, map_file=None): """Display final information about a generated archive diff -Nru sosreport-4.2/sos/policies/init_systems/__init__.py sosreport-4.3/sos/policies/init_systems/__init__.py --- sosreport-4.2/sos/policies/init_systems/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/init_systems/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -29,9 +29,14 @@ status of services :type query_cmd: ``str`` + :param chroot: Location to chroot to for any command execution, i.e. the + sysroot if we're running in a container + :type chroot: ``str`` or ``None`` + """ - def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None): + def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None, + chroot=None): """Initialize a new InitSystem()""" self.services = {} @@ -39,6 +44,7 @@ self.init_cmd = init_cmd self.list_cmd = "%s %s" % (self.init_cmd, list_cmd) or None self.query_cmd = "%s %s" % (self.init_cmd, query_cmd) or None + self.chroot = chroot def is_enabled(self, name): """Check if given service name is enabled @@ -108,7 +114,10 @@ """Query an individual service""" if self.query_cmd: try: - return sos_get_command_output("%s %s" % (self.query_cmd, name)) + return sos_get_command_output( + "%s %s" % (self.query_cmd, name), + chroot=self.chroot + ) except Exception: return None return None diff -Nru sosreport-4.2/sos/policies/init_systems/systemd.py sosreport-4.3/sos/policies/init_systems/systemd.py --- sosreport-4.2/sos/policies/init_systems/systemd.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/init_systems/systemd.py 2022-02-15 04:20:20.000000000 +0000 @@ -15,11 +15,12 @@ class SystemdInit(InitSystem): """InitSystem abstraction for SystemD systems""" - def __init__(self): + def __init__(self, chroot=None): super(SystemdInit, self).__init__( init_cmd='systemctl', list_cmd='list-unit-files --type=service', - query_cmd='status' + query_cmd='status', + chroot=chroot ) self.load_all_services() @@ -30,7 +31,7 @@ return 'unknown' def load_all_services(self): - svcs = shell_out(self.list_cmd).splitlines()[1:] + svcs = shell_out(self.list_cmd, chroot=self.chroot).splitlines()[1:] for line in svcs: try: name = line.split('.service')[0] diff -Nru sosreport-4.2/sos/policies/runtimes/crio.py sosreport-4.3/sos/policies/runtimes/crio.py --- sosreport-4.2/sos/policies/runtimes/crio.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/policies/runtimes/crio.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,82 @@ +# Copyright (C) 2021 Red Hat, Inc., Nadia Pinaeva + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.policies.runtimes import ContainerRuntime +from sos.utilities import sos_get_command_output +from pipes import quote + + +class CrioContainerRuntime(ContainerRuntime): + """Runtime class to use for systems running crio""" + + name = 'crio' + binary = 'crictl' + + def check_can_copy(self): + return False + + def get_containers(self, get_all=False): + """Get a list of containers present on the system. + + :param get_all: If set, include stopped containers as well + :type get_all: ``bool`` + """ + containers = [] + _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '') + if self.active: + out = sos_get_command_output(_cmd, chroot=self.policy.sysroot) + if out['status'] == 0: + for ent in out['output'].splitlines()[1:]: + ent = ent.split() + # takes the form (container_id, container_name) + containers.append((ent[0], ent[-3])) + return containers + + def get_images(self): + """Get a list of images present on the system + + :returns: A list of 2-tuples containing (image_name, image_id) + :rtype: ``list`` + """ + images = [] + if self.active: + out = sos_get_command_output("%s images" % self.binary, + chroot=self.policy.sysroot) + if out['status'] == 0: + for ent in out['output'].splitlines(): + ent = ent.split() + # takes the form (image_name, image_id) + images.append((ent[0] + ':' + ent[1], ent[2])) + return images + + def fmt_container_cmd(self, container, cmd, quotecmd): + """Format a command to run inside a container using the runtime + + :param container: The name or ID of the container in which to run + :type container: ``str`` + + :param cmd: The command to run inside `container` + :type cmd: ``str`` + + :param quotecmd: Whether the cmd should be quoted. + :type quotecmd: ``bool`` + + :returns: Formatted string to run `cmd` inside `container` + :rtype: ``str`` + """ + if quotecmd: + quoted_cmd = quote(cmd) + else: + quoted_cmd = cmd + container_id = self.get_container_by_name(container) + return "%s %s %s" % (self.run_cmd, container_id, + quoted_cmd) if container_id is not None else '' + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/policies/runtimes/docker.py sosreport-4.3/sos/policies/runtimes/docker.py --- sosreport-4.2/sos/policies/runtimes/docker.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/runtimes/docker.py 2022-02-15 04:20:20.000000000 +0000 @@ -18,13 +18,16 @@ name = 'docker' binary = 'docker' - def check_is_active(self): + def check_is_active(self, sysroot=None): # the daemon must be running - if (is_executable('docker') and + if (is_executable('docker', sysroot) and (self.policy.init_system.is_running('docker') or self.policy.init_system.is_running('snap.docker.dockerd'))): self.active = True return True return False + def check_can_copy(self): + return self.check_is_active(sysroot=self.policy.sysroot) + # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/policies/runtimes/__init__.py sosreport-4.3/sos/policies/runtimes/__init__.py --- sosreport-4.2/sos/policies/runtimes/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/policies/runtimes/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -64,11 +64,17 @@ :returns: ``True`` if the runtime is active, else ``False`` :rtype: ``bool`` """ - if is_executable(self.binary): + if is_executable(self.binary, self.policy.sysroot): self.active = True return True return False + def check_can_copy(self): + """Check if the runtime supports copying files out of containers and + onto the host filesystem + """ + return True + def get_containers(self, get_all=False): """Get a list of containers present on the system. @@ -78,7 +84,7 @@ containers = [] _cmd = "%s ps %s" % (self.binary, '-a' if get_all else '') if self.active: - out = sos_get_command_output(_cmd) + out = sos_get_command_output(_cmd, chroot=self.policy.sysroot) if out['status'] == 0: for ent in out['output'].splitlines()[1:]: ent = ent.split() @@ -100,7 +106,7 @@ return None for c in self.containers: if re.match(name, c[1]): - return c[1] + return c[0] return None def get_images(self): @@ -112,8 +118,10 @@ images = [] fmt = '{{lower .Repository}}:{{lower .Tag}} {{lower .ID}}' if self.active: - out = sos_get_command_output("%s images --format '%s'" - % (self.binary, fmt)) + out = sos_get_command_output( + "%s images --format '%s'" % (self.binary, fmt), + chroot=self.policy.sysroot + ) if out['status'] == 0: for ent in out['output'].splitlines(): ent = ent.split() @@ -129,7 +137,10 @@ """ vols = [] if self.active: - out = sos_get_command_output("%s volume ls" % self.binary) + out = sos_get_command_output( + "%s volume ls" % self.binary, + chroot=self.policy.sysroot + ) if out['status'] == 0: for ent in out['output'].splitlines()[1:]: ent = ent.split() @@ -194,5 +205,31 @@ """ return "%s logs -t %s" % (self.binary, container) + def get_copy_command(self, container, path, dest, sizelimit=None): + """Generate the command string used to copy a file out of a container + by way of the runtime. + + :param container: The name or ID of the container + :type container: ``str`` + + :param path: The path to copy from the container. Note that at + this time, no supported runtime supports globbing + :type path: ``str`` + + :param dest: The destination on the *host* filesystem to write + the file to + :type dest: ``str`` + + :param sizelimit: Limit the collection to the last X bytes of the + file at PATH + :type sizelimit: ``int`` + + :returns: Formatted runtime command to copy a file from a container + :rtype: ``str`` + """ + if sizelimit: + return "%s %s tail -c %s %s" % (self.run_cmd, container, sizelimit, + path) + return "%s cp %s:%s %s" % (self.binary, container, path, dest) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/presets/redhat/__init__.py sosreport-4.3/sos/presets/redhat/__init__.py --- sosreport-4.2/sos/presets/redhat/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/presets/redhat/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -29,11 +29,15 @@ RHOSP = "rhosp" RHOSP_DESC = "Red Hat OpenStack Platform" +RHOSP_OPTS = SoSOptions(plugopts=[ + 'process.lsof=off', + 'networking.ethtool_namespaces=False', + 'networking.namespaces=200']) RHOCP = "ocp" RHOCP_DESC = "OpenShift Container Platform by Red Hat" -RHOSP_OPTS = SoSOptions(plugopts=[ - 'process.lsof=off', +RHOCP_OPTS = SoSOptions(all_logs=True, verify=True, plugopts=[ + 'networking.timeout=600', 'networking.ethtool_namespaces=False', 'networking.namespaces=200']) @@ -62,7 +66,7 @@ RHEL: PresetDefaults(name=RHEL, desc=RHEL_DESC), RHOSP: PresetDefaults(name=RHOSP, desc=RHOSP_DESC, opts=RHOSP_OPTS), RHOCP: PresetDefaults(name=RHOCP, desc=RHOCP_DESC, note=NOTE_SIZE_TIME, - opts=_opts_all_logs_verify), + opts=RHOCP_OPTS), RH_CFME: PresetDefaults(name=RH_CFME, desc=RH_CFME_DESC, note=NOTE_TIME, opts=_opts_verify), RH_SATELLITE: PresetDefaults(name=RH_SATELLITE, desc=RH_SATELLITE_DESC, diff -Nru sosreport-4.2/sos/report/__init__.py sosreport-4.3/sos/report/__init__.py --- sosreport-4.2/sos/report/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -17,7 +17,7 @@ from datetime import datetime import glob import sos.report.plugins -from sos.utilities import (ImporterHelper, SoSTimeoutError, +from sos.utilities import (ImporterHelper, SoSTimeoutError, bold, sos_get_command_output, TIMEOUT_DEFAULT) from shutil import rmtree import hashlib @@ -82,10 +82,12 @@ 'case_id': '', 'chroot': 'auto', 'clean': False, + 'container_runtime': 'auto', 'keep_binary_files': False, 'desc': '', 'domains': [], 'dry_run': False, + 'estimate_only': False, 'experimental': False, 'enable_plugins': [], 'keywords': [], @@ -137,6 +139,7 @@ self._args = args self.sysroot = "/" self.preset = None + self.estimated_plugsizes = {} self.print_header() self._set_debug() @@ -171,14 +174,12 @@ self._set_directories() msg = "default" - host_sysroot = self.policy.host_sysroot() + self.sysroot = self.policy.sysroot # set alternate system root directory if self.opts.sysroot: msg = "cmdline" - self.sysroot = self.opts.sysroot - elif self.policy.in_container() and host_sysroot != os.sep: + elif self.policy.in_container() and self.sysroot != os.sep: msg = "policy" - self.sysroot = host_sysroot self.soslog.debug("set sysroot to '%s' (%s)" % (self.sysroot, msg)) if self.opts.chroot not in chroot_modes: @@ -187,6 +188,7 @@ self.tempfile_util.clean() self._exit(1) + self._check_container_runtime() self._get_hardware_devices() self._get_namespaces() @@ -218,11 +220,19 @@ dest="chroot", default='auto', help="chroot executed commands to SYSROOT " "[auto, always, never] (default=auto)") + report_grp.add_argument("--container-runtime", default="auto", + help="Default container runtime to use for " + "collections. 'auto' for policy control.") report_grp.add_argument("--desc", "--description", type=str, action="store", default="", help="Description for a new preset",) report_grp.add_argument("--dry-run", action="store_true", help="Run plugins but do not collect data") + report_grp.add_argument("--estimate-only", action="store_true", + help="Approximate disk space requirements for " + "a real sos run; disables --clean and " + "--collect, sets --threads=1 and " + "--no-postproc") report_grp.add_argument("--experimental", action="store_true", dest="experimental", default=False, help="enable experimental plugins") @@ -332,7 +342,7 @@ cleaner_grp.add_argument('--clean', '--cleaner', '--mask', dest='clean', default=False, action='store_true', - help='Obfuscate sensistive information') + help='Obfuscate sensitive information') cleaner_grp.add_argument('--domains', dest='domains', default=[], action='extend', help='Additional domain names to obfuscate') @@ -358,6 +368,62 @@ action='extend', help='List of usernames to obfuscate') + @classmethod + def display_help(cls, section): + section.set_title('SoS Report Detailed Help') + section.add_text( + 'The report command is the most common use case for SoS, and aims ' + 'to collect relevant diagnostic and troubleshooting data to assist' + ' with issue analysis without actively performing that analysis on' + ' the system while it is in use.' + ) + section.add_text( + 'Additionally, sos report archives can be used for ongoing ' + 'inspection for pre-emptive issue monitoring, such as that done ' + 'by the Insights project.' + ) + + section.add_text( + 'The typical result of an execution of \'sos report\' is a tarball' + ' that contains troubleshooting command output, copies of config ' + 'files, and copies of relevant sections of the host filesystem. ' + 'Root privileges are required for collections.' + ) + + psec = section.add_section(title='How Collections Are Determined') + psec.add_text( + 'SoS report performs it\'s collections by way of \'plugins\' that ' + 'individually specify what files to copy and what commands to run.' + ' Plugins typically map to specific components or software ' + 'packages.' + ) + psec.add_text( + 'Plugins may specify different collections on different distribu' + 'tions, and some plugins may only be for specific distributions. ' + 'Distributions are represented within SoS by \'policies\' and may ' + 'influence how other SoS commands or options function. For example' + 'policies can alter where the --upload option defaults to or ' + 'functions.' + ) + + ssec = section.add_section(title='See Also') + ssec.add_text( + "For information on available options for report, see %s and %s" + % (bold('sos report --help'), bold('man sos-report')) + ) + ssec.add_text("The following %s sections may be of interest:\n" + % bold('sos help')) + help_lines = { + 'report.plugins': 'Information on the plugin design of sos', + 'report.plugins.$plugin': 'Information on a specific $plugin', + 'policies': 'How sos operates on different distributions' + } + helpln = '' + for ln in help_lines: + ssec.add_text("\t{:<36}{}".format(ln, help_lines[ln]), + newline=False) + ssec.add_text(helpln) + def print_header(self): print("\n%s\n" % _("sosreport (version %s)" % (__version__,))) @@ -368,6 +434,37 @@ } # TODO: enumerate network devices, preferably with devtype info + def _check_container_runtime(self): + """Check the loaded container runtimes, and the policy default runtime + (if set), against any requested --container-runtime value. This can be + useful for systems that have multiple runtimes, such as RHCOS, but do + not have a clearly defined 'default' (or one that is determined based + entirely on configuration). + """ + if self.opts.container_runtime != 'auto': + crun = self.opts.container_runtime.lower() + if crun in ['none', 'off', 'diabled']: + self.policy.runtimes = {} + self.soslog.info( + "Disabled all container runtimes per user option." + ) + elif not self.policy.runtimes: + msg = ("WARNING: No container runtimes are active, ignoring " + "option to set default runtime to '%s'\n" % crun) + self.soslog.warn(msg) + elif crun not in self.policy.runtimes.keys(): + valid = ', '.join(p for p in self.policy.runtimes.keys() + if p != 'default') + raise Exception("Cannot use container runtime '%s': no such " + "runtime detected. Available runtimes: %s" + % (crun, valid)) + else: + self.policy.runtimes['default'] = self.policy.runtimes[crun] + self.soslog.info( + "Set default container runtime to '%s'" + % self.policy.runtimes['default'].name + ) + def get_fibre_devs(self): """Enumerate a list of fibrechannel devices on this system so that plugins can iterate over them @@ -619,21 +716,24 @@ def _set_all_options(self): if self.opts.alloptions: for plugname, plug in self.loaded_plugins: - for name, parms in zip(plug.opt_names, plug.opt_parms): - if type(parms["enabled"]) == bool: - parms["enabled"] = True + for opt in plug.options.values(): + if bool in opt.val_type: + opt.value = True def _set_tunables(self): if self.opts.plugopts: opts = {} for opt in self.opts.plugopts: - # split up "general.syslogsize=5" try: opt, val = opt.split("=") except ValueError: val = True - else: - if val.lower() in ["off", "disable", "disabled", "false"]: + + if isinstance(val, str): + val = val.lower() + if val in ["on", "enable", "enabled", "true", "yes"]: + val = True + elif val in ["off", "disable", "disabled", "false", "no"]: val = False else: # try to convert string "val" to int() @@ -642,7 +742,6 @@ except ValueError: pass - # split up "general.syslogsize" try: plug, opt = opt.split(".") except ValueError: @@ -652,16 +751,25 @@ try: opts[plug] except KeyError: - opts[plug] = [] - opts[plug].append((opt, val)) + opts[plug] = {} + opts[plug][opt] = val for plugname, plug in self.loaded_plugins: if plugname in opts: - for opt, val in opts[plugname]: - if not plug.set_option(opt, val): + for opt in opts[plugname]: + if opt not in plug.options: self.soslog.error('no such option "%s" for plugin ' '(%s)' % (opt, plugname)) self._exit(1) + try: + plug.options[opt].set_value(opts[plugname][opt]) + self.soslog.debug( + "Set %s plugin option to %s" + % (plugname, plug.options[opt]) + ) + except Exception as err: + self.soslog.error(err) + self._exit(1) del opts[plugname] for plugname in opts.keys(): self.soslog.error('WARNING: unable to set option for disabled ' @@ -688,10 +796,35 @@ def _set_plugin_options(self): for plugin_name, plugin in self.loaded_plugins: - names, parms = plugin.get_all_options() - for optname, optparm in zip(names, parms): - self.all_options.append((plugin, plugin_name, optname, - optparm)) + for opt in plugin.options: + self.all_options.append(plugin.options[opt]) + + def _set_estimate_only(self): + # set estimate-only mode by enforcing some options settings + # and return a corresponding log messages string + msg = "\nEstimate-only mode enabled" + ext_msg = [] + if self.opts.threads > 1: + ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ] + self.opts.threads = 1 + if not self.opts.build: + ext_msg += ["--build enabled", ] + self.opts.build = True + if not self.opts.no_postproc: + ext_msg += ["--no-postproc enabled", ] + self.opts.no_postproc = True + if self.opts.clean: + ext_msg += ["--clean disabled", ] + self.opts.clean = False + if self.opts.upload: + ext_msg += ["--upload* options disabled", ] + self.opts.upload = False + if ext_msg: + msg += ", which overrides some options:\n " + "\n ".join(ext_msg) + else: + msg += "." + msg += "\n\n" + return msg def _report_profiles_and_plugins(self): self.ui_log.info("") @@ -732,31 +865,33 @@ if self.all_options: self.ui_log.info(_("The following options are available for ALL " "plugins:")) - for opt in self.all_options[0][0]._default_plug_opts: - val = opt[3] - if val == -1: + _defaults = self.loaded_plugins[0][1].get_default_plugin_opts() + for _opt in _defaults: + opt = _defaults[_opt] + val = opt.default + if opt.default == -1: val = TIMEOUT_DEFAULT - self.ui_log.info(" %-25s %-15s %s" % (opt[0], val, opt[1])) + self.ui_log.info(" %-25s %-15s %s" % (opt.name, val, opt.desc)) self.ui_log.info("") self.ui_log.info(_("The following plugin options are available:")) - for (plug, plugname, optname, optparm) in self.all_options: - if optname in ('timeout', 'postproc', 'cmd-timeout'): + for opt in self.all_options: + if opt.name in ('timeout', 'postproc', 'cmd-timeout'): continue # format option value based on its type (int or bool) - if type(optparm["enabled"]) == bool: - if optparm["enabled"] is True: + if isinstance(opt.default, bool): + if opt.default is True: tmpopt = "on" else: tmpopt = "off" else: - tmpopt = optparm["enabled"] + tmpopt = opt.default if tmpopt is None: tmpopt = 0 self.ui_log.info(" %-25s %-15s %s" % ( - plugname + "." + optname, tmpopt, optparm["desc"])) + opt.plugin + "." + opt.name, tmpopt, opt.desc)) else: self.ui_log.info(_("No plugin options available.")) @@ -864,10 +999,12 @@ return True def batch(self): + msg = self.policy.get_msg() + if self.opts.estimate_only: + msg += self._set_estimate_only() if self.opts.batch: - self.ui_log.info(self.policy.get_msg()) + self.ui_log.info(msg) else: - msg = self.policy.get_msg() msg += _("Press ENTER to continue, or CTRL-C to quit.\n") try: input(msg) @@ -875,9 +1012,7 @@ self.ui_log.error("Exiting on user cancel") self._exit(130) except Exception as e: - self.ui_log.info("") - self.ui_log.error(e) - self._exit(e) + self._exit(1, e) def _log_plugin_exception(self, plugin, method): trace = traceback.format_exc() @@ -914,20 +1049,6 @@ self._exit(1) def setup(self): - # Log command line options - msg = "[%s:%s] executing 'sos %s'" - self.soslog.info(msg % (__name__, "setup", " ".join(self.cmdline))) - - # Log active preset defaults - preset_args = self.preset.opts.to_args() - msg = ("[%s:%s] using '%s' preset defaults (%s)" % - (__name__, "setup", self.preset.name, " ".join(preset_args))) - self.soslog.info(msg) - - # Log effective options after applying preset defaults - self.soslog.info("[%s:%s] effective options now: %s" % - (__name__, "setup", " ".join(self.opts.to_args()))) - self.ui_log.info(_(" Setting up plugins ...")) for plugname, plug in self.loaded_plugins: try: @@ -1010,10 +1131,31 @@ _plug.manifest.add_field('end_time', end) _plug.manifest.add_field('run_time', end - start) except TimeoutError: - self.ui_log.error("\n Plugin %s timed out\n" % plugin[1]) + msg = "Plugin %s timed out" % plugin[1] + # log to ui_log.error to show the user, log to soslog.info + # so that someone investigating the sos execution has it all + # in one place, but without double notifying the user. + self.ui_log.error("\n %s\n" % msg) + self.soslog.info(msg) self.running_plugs.remove(plugin[1]) self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() + pool.shutdown(wait=True) pool._threads.clear() + if self.opts.estimate_only: + from pathlib import Path + tmpdir_path = Path(self.archive.get_tmp_dir()) + self.estimated_plugsizes[plugin[1]] = sum( + [f.lstat().st_size for f in tmpdir_path.glob('**/*')]) + # remove whole tmp_dir content - including "sos_commands" and + # similar dirs that will be re-created on demand by next plugin + # if needed; it is less error-prone approach than skipping + # deletion of some dirs but deleting their content + for f in os.listdir(self.archive.get_tmp_dir()): + f = os.path.join(self.archive.get_tmp_dir(), f) + if os.path.isdir(f) and not os.path.islink(f): + rmtree(f) + else: + os.unlink(f) return True def collect_plugin(self, plugin): @@ -1113,13 +1255,9 @@ cmd['file'] ))) - for content, f in plug.copy_strings: + for content, f, tags in plug.copy_strings: section.add(CreatedFile(name=f, - href=os.path.join( - "..", - "sos_strings", - plugname, - f))) + href=os.path.join("..", f))) report.add(section) @@ -1229,6 +1367,34 @@ short_name='manifest.json' ) + # print results in estimate mode (to include also just added manifest) + if self.opts.estimate_only: + from sos.utilities import get_human_readable + from pathlib import Path + # add sos_logs, sos_reports dirs, etc., basically everything + # that remained in self.tmpdir after plugins' contents removal + # that still will be moved to the sos report final directory path + tmpdir_path = Path(self.tmpdir) + self.estimated_plugsizes['sos_logs_reports'] = sum( + [f.lstat().st_size for f in tmpdir_path.glob('**/*')]) + + _sum = get_human_readable(sum(self.estimated_plugsizes.values())) + self.ui_log.info("Estimated disk space requirement for whole " + "uncompressed sos report directory: %s" % _sum) + bigplugins = sorted(self.estimated_plugsizes.items(), + key=lambda x: x[1], reverse=True)[:5] + bp_out = ", ".join("%s: %s" % + (p, get_human_readable(v, precision=0)) + for p, v in bigplugins) + self.ui_log.info("Five biggest plugins: %s" % bp_out) + self.ui_log.info("") + self.ui_log.info("Please note the estimation is relevant to the " + "current options.") + self.ui_log.info("Be aware that the real disk space requirements " + "might be different. A rule of thumb is to " + "reserve at least double the estimation.") + self.ui_log.info("") + # package up and compress the results if not self.opts.build: old_umask = os.umask(0o077) @@ -1375,11 +1541,27 @@ self.report_md.add_list('disabled_plugins', self.opts.skip_plugins) self.report_md.add_section('plugins') + def _merge_preset_options(self): + # Log command line options + msg = "[%s:%s] executing 'sos %s'" + self.soslog.info(msg % (__name__, "setup", " ".join(self.cmdline))) + + # Log active preset defaults + preset_args = self.preset.opts.to_args() + msg = ("[%s:%s] using '%s' preset defaults (%s)" % + (__name__, "setup", self.preset.name, " ".join(preset_args))) + self.soslog.info(msg) + + # Log effective options after applying preset defaults + self.soslog.info("[%s:%s] effective options now: %s" % + (__name__, "setup", " ".join(self.opts.to_args()))) + def execute(self): try: self.policy.set_commons(self.get_commons()) self.load_plugins() self._set_all_options() + self._merge_preset_options() self._set_tunables() self._check_for_unknown_plugins() self._set_plugin_options() @@ -1420,13 +1602,15 @@ except (OSError): if self.opts.debug: raise - self.cleanup() + if not os.getenv('SOS_TEST_LOGS', None) == 'keep': + self.cleanup() except (KeyboardInterrupt): self.ui_log.error("\nExiting on user cancel") self.cleanup() self._exit(130) except (SystemExit) as e: - self.cleanup() + if not os.getenv('SOS_TEST_LOGS', None) == 'keep': + self.cleanup() sys.exit(e.code) self._exit(1) diff -Nru sosreport-4.2/sos/report/plugins/abrt.py sosreport-4.3/sos/report/plugins/abrt.py --- sosreport-4.2/sos/report/plugins/abrt.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/abrt.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Abrt(Plugin, RedHatPlugin): @@ -21,7 +21,8 @@ files = ('/var/spool/abrt',) option_list = [ - ("detailed", 'collect detailed info for every report', 'slow', False) + PluginOpt("detailed", default=False, + desc="collect detailed information for every report") ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/apache.py sosreport-4.3/sos/report/plugins/apache.py --- sosreport-4.2/sos/report/plugins/apache.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/apache.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,10 +6,24 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) class Apache(Plugin): + """The Apache plugin covers the upstream Apache webserver project, + regardless of the packaged name; apache2 for Debian and Ubuntu, or httpd + for Red Hat family distributions. + + The aim of this plugin is for Apache-specific information, not necessarily + other projects that happen to place logs or similar files within the + standardized apache directories. For example, OpenStack components that log + to apache logging directories are excluded from this plugin and collected + via their respective OpenStack plugins. + + Users can expect the collection of apachectl command output, apache server + logs, and apache configuration files from this plugin. + """ short_desc = 'Apache http daemon' plugin_name = "apache" @@ -18,7 +32,7 @@ files = ('/var/www/',) option_list = [ - ("log", "gathers all apache logs", "slow", False) + PluginOpt(name="log", default=False, desc="gathers all apache logs") ] def setup(self): @@ -48,6 +62,15 @@ class RedHatApache(Apache, RedHatPlugin): + """ + On Red Hat distributions, the Apache plugin will also attempt to collect + JBoss Web Server logs and configuration files. + + Note that for Red Hat distributions, this plugin explicitly collects for + 'httpd' installations. If you have installed apache from source or via any + method that uses the name 'apache' instead of 'httpd', these collections + will fail. + """ files = ( '/etc/httpd/conf/httpd.conf', '/etc/httpd22/conf/httpd.conf', diff -Nru sosreport-4.2/sos/report/plugins/ata.py sosreport-4.3/sos/report/plugins/ata.py --- sosreport-4.2/sos/report/plugins/ata.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ata.py 2022-02-15 04:20:20.000000000 +0000 @@ -23,7 +23,9 @@ cmd_list = [ "hdparm %(dev)s", "smartctl -a %(dev)s", - "smartctl -l scterc %(dev)s" + "smartctl -a %(dev)s -j", + "smartctl -l scterc %(dev)s", + "smartctl -l scterc %(dev)s -j" ] self.add_blockdev_cmd(cmd_list, whitelist=['sd.*', 'hd.*']) diff -Nru sosreport-4.2/sos/report/plugins/atomichost.py sosreport-4.3/sos/report/plugins/atomichost.py --- sosreport-4.2/sos/report/plugins/atomichost.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/atomichost.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class AtomicHost(Plugin, RedHatPlugin): @@ -18,7 +18,8 @@ plugin_name = "atomichost" profiles = ('container',) option_list = [ - ("info", "gather atomic info for each image", "fast", False) + PluginOpt("info", default=False, + desc="gather atomic info for each image") ] def check_enabled(self): diff -Nru sosreport-4.2/sos/report/plugins/azure.py sosreport-4.3/sos/report/plugins/azure.py --- sosreport-4.2/sos/report/plugins/azure.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/azure.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,8 +8,8 @@ # # See the LICENSE file in the source distribution for further information. -import os from sos.report.plugins import Plugin, UbuntuPlugin, RedHatPlugin +import os class Azure(Plugin, UbuntuPlugin): @@ -38,7 +38,7 @@ for path, subdirs, files in os.walk("/var/log/azure"): for name in files: - self.add_copy_spec(os.path.join(path, name), sizelimit=limit) + self.add_copy_spec(self.path_join(path, name), sizelimit=limit) self.add_cmd_output(( 'curl -s -H Metadata:true ' diff -Nru sosreport-4.2/sos/report/plugins/boot.py sosreport-4.3/sos/report/plugins/boot.py --- sosreport-4.2/sos/report/plugins/boot.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/boot.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt from glob import glob @@ -19,7 +19,8 @@ packages = ('grub', 'grub2', 'grub-common', 'grub2-common', 'zipl') option_list = [ - ("all-images", "collect lsinitrd for all images", "slow", False) + PluginOpt("all-images", default=False, + desc="collect lsinitrd for all images") ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/ceph_common.py sosreport-4.3/sos/report/plugins/ceph_common.py --- sosreport-4.2/sos/report/plugins/ceph_common.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ceph_common.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,85 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +from socket import gethostname + + +class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = 'CEPH common' + + plugin_name = 'ceph_common' + profiles = ('storage', 'virt', 'container') + + containers = ('ceph-(mon|rgw|osd).*',) + ceph_hostname = gethostname() + + packages = ( + 'ceph', + 'ceph-mds', + 'ceph-common', + 'libcephfs1', + 'ceph-fs-common', + 'calamari-server', + 'librados2' + ) + + services = ( + 'ceph-nfs@pacemaker', + 'ceph-mds@%s' % ceph_hostname, + 'ceph-mon@%s' % ceph_hostname, + 'ceph-mgr@%s' % ceph_hostname, + 'ceph-radosgw@*', + 'ceph-osd@*' + ) + + # This check will enable the plugin regardless of being + # containerized or not + files = ('/etc/ceph/ceph.conf',) + + def setup(self): + all_logs = self.get_option("all_logs") + + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + '/var/log/ceph/ceph.log.*': 'ceph_log', + }) + + if not all_logs: + self.add_copy_spec("/var/log/calamari/*.log",) + else: + self.add_copy_spec("/var/log/calamari",) + + self.add_copy_spec([ + "/var/log/ceph/ceph.log", + "/var/log/ceph/ceph.audit.log*", + "/var/log/calamari/*.log", + "/etc/ceph/", + "/etc/calamari/", + "/var/lib/ceph/tmp/", + ]) + + self.add_cmd_output([ + "ceph -v", + ]) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/ceph_mds.py sosreport-4.3/sos/report/plugins/ceph_mds.py --- sosreport-4.2/sos/report/plugins/ceph_mds.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ceph_mds.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,97 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +import glob + + +class CephMDS(Plugin, RedHatPlugin, UbuntuPlugin): + short_desc = 'CEPH mds' + plugin_name = 'ceph_mds' + profiles = ('storage', 'virt', 'container') + containers = ('ceph-fs.*',) + + def check_enabled(self): + return True if glob.glob('/var/lib/ceph/mds/*/*') else False + + def setup(self): + self.add_file_tags({ + '/var/log/ceph/ceph-mds.*.log': 'ceph_mds_log', + }) + + self.add_copy_spec([ + "/var/log/ceph/ceph-mds*.log", + "/var/lib/ceph/bootstrap-mds/", + "/var/lib/ceph/mds/", + "/run/ceph/ceph-mds*", + ]) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + ceph_cmds = [ + "cache status", + "client ls", + "config diff", + "config show", + "damage ls", + "dump loads", + "dump tree", + "dump_blocked_ops", + "dump_historic_ops", + "dump_historic_ops_by_duration", + "dump_mempools", + "dump_ops_in_flight", + "get subtrees", + "objecter_requests", + "ops", + "perf histogram dump", + "perf histogram schema", + "perf schema", + "perf dump", + "status", + "version", + "session ls" + ] + + mds_ids = [] + # Get the ceph user processes + out = self.exec_cmd('ps -u ceph -o args') + + if out['status'] == 0: + # Extract the OSD ids from valid output lines + for procs in out['output'].splitlines(): + proc = procs.split() + if len(proc) < 6: + continue + if proc[4] == '--id' and "ceph-mds" in proc[0]: + mds_ids.append("mds.%s" % proc[5]) + + # If containerized, run commands in containers + try: + cname = self.get_all_containers_by_regex("ceph-mds*")[0][1] + except Exception: + cname = None + + self.add_cmd_output([ + "ceph daemon %s %s" + % (mdsid, cmd) for mdsid in mds_ids for cmd in ceph_cmds + ], container=cname) + + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/ceph_mgr.py sosreport-4.3/sos/report/plugins/ceph_mgr.py --- sosreport-4.2/sos/report/plugins/ceph_mgr.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ceph_mgr.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,98 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +import glob + + +class CephMGR(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = 'CEPH mgr' + + plugin_name = 'ceph_mgr' + profiles = ('storage', 'virt', 'container') + + containers = ('ceph-mgr.*',) + + def check_enabled(self): + return True if glob.glob('/var/lib/ceph/mgr/*/*') else False + + def setup(self): + self.add_file_tags({ + '/var/log/ceph/ceph-mgr.*.log': 'ceph_mgr_log', + }) + + self.add_copy_spec([ + "/var/log/ceph/ceph-mgr*.log", + "/var/lib/ceph/mgr/", + "/var/lib/ceph/bootstrap-mgr/", + "/run/ceph/ceph-mgr*", + ]) + + # more commands to be added later + self.add_cmd_output([ + "ceph balancer status", + ]) + + ceph_cmds = [ + "config diff", + "config show", + "dump_cache", + "dump_mempools", + "dump_osd_network", + "mds_requests", + "mds_sessions", + "objecter_requests", + "mds_requests", + "mds_sessions", + "perf dump", + "perf histogram dump", + "perf histogram schema", + "perf schema", + "status", + "version" + ] + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*", + ]) + + mgr_ids = [] + # Get the ceph user processes + out = self.exec_cmd('ps -u ceph -o args') + + if out['status'] == 0: + # Extract the OSD ids from valid output lines + for procs in out['output'].splitlines(): + proc = procs.split() + if len(proc) < 6: + continue + if proc[4] == '--id' and "ceph-mgr" in proc[0]: + mgr_ids.append("mgr.%s" % proc[5]) + + # If containerized, run commands in containers + try: + cname = self.get_all_containers_by_regex("ceph-mgr*")[0][1] + except Exception: + cname = None + + self.add_cmd_output([ + "ceph daemon %s %s" + % (mgrid, cmd) for mgrid in mgr_ids for cmd in ceph_cmds + ], container=cname) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/ceph_mon.py sosreport-4.3/sos/report/plugins/ceph_mon.py --- sosreport-4.2/sos/report/plugins/ceph_mon.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ceph_mon.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,118 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +import glob + + +class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = 'CEPH mon' + + plugin_name = 'ceph_mon' + profiles = ('storage', 'virt', 'container') + containers = ('ceph-mon.*',) + + def check_enabled(self): + return True if glob.glob('/var/lib/ceph/mon/*/*') else False + + def setup(self): + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + '/var/log/ceph/ceph-mon.*.log': 'ceph_mon_log' + }) + + self.add_copy_spec([ + "/var/log/ceph/ceph-mon*.log", + "/var/lib/ceph/mon/", + "/run/ceph/ceph-mon*" + ]) + + self.add_cmd_output([ + # The ceph_mon plugin will collect all the "ceph ..." commands + # which typically require the keyring. + + "ceph mon stat", + "ceph quorum_status", + "ceph report", + "ceph-disk list", + "ceph versions", + "ceph features", + "ceph insights", + "ceph crash stat", + "ceph crash ls", + "ceph config log", + "ceph config generate-minimal-conf", + "ceph config-key dump", + "ceph mon_status", + "ceph osd metadata", + "ceph osd erasure-code-profile ls", + "ceph osd crush show-tunables", + "ceph osd crush dump", + "ceph mgr dump", + "ceph mgr metadata", + "ceph mgr module ls", + "ceph mgr services", + "ceph mgr versions" + ]) + + ceph_cmds = [ + "mon dump", + "status", + "health detail", + "device ls", + "df", + "df detail", + "fs ls", + "fs dump", + "pg dump", + "pg stat", + "time-sync-status", + "osd tree", + "osd stat", + "osd df tree", + "osd dump", + "osd df", + "osd perf", + "osd blocked-by", + "osd pool ls detail", + "osd pool autoscale-status", + "mds stat", + "osd numa-status" + ] + + self.add_cmd_output([ + "ceph %s --format json-pretty" % s for s in ceph_cmds + ], subdir="json_output", tags="insights_ceph_health_detail") + + # these can be cleaned up too but leaving them for safety for now + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + # If containerized, run commands in containers + try: + cname = self.get_all_containers_by_regex("ceph-mon*")[0][1] + except Exception: + cname = None + + self.add_cmd_output( + ["ceph %s" % cmd for cmd in ceph_cmds], + container=cname + ) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/ceph_osd.py sosreport-4.3/sos/report/plugins/ceph_osd.py --- sosreport-4.2/sos/report/plugins/ceph_osd.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ceph_osd.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,104 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +import glob + + +class CephOSD(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = 'CEPH osd' + + plugin_name = 'ceph_osd' + profiles = ('storage', 'virt', 'container') + containers = ('ceph-osd.*',) + + def check_enabled(self): + return True if glob.glob('/var/lib/ceph/osd/*/*') else False + + def setup(self): + self.add_file_tags({ + '/var/log/ceph/ceph-osd.*.log': 'ceph_osd_log', + }) + + # Only collect OSD specific files + self.add_copy_spec([ + "/var/log/ceph/ceph-osd*.log", + "/var/log/ceph/ceph-volume*.log", + + "/var/lib/ceph/osd/", + "/var/lib/ceph/bootstrap-osd/", + + "/run/ceph/ceph-osd*" + ]) + + self.add_cmd_output([ + "ceph-disk list", + "ceph-volume lvm list" + ]) + + ceph_cmds = [ + "bluestore bluefs available", + "config diff", + "config show", + "dump_blacklist", + "dump_blocked_ops", + "dump_historic_ops_by_duration", + "dump_historic_slow_ops", + "dump_mempools", + "dump_ops_in_flight", + "dump_op_pq_state", + "dump_osd_network", + "dump_reservations", + "dump_watchers", + "log dump", + "perf dump", + "perf histogram dump", + "objecter_requests", + "ops", + "status", + "version", + ] + + osd_ids = [] + # Get the ceph user processes + out = self.exec_cmd('ps -u ceph -o args') + + if out['status'] == 0: + # Extract the OSD ids from valid output lines + for procs in out['output'].splitlines(): + proc = procs.split() + if len(proc) < 6: + continue + if proc[4] == '--id' and proc[5].isdigit(): + osd_ids.append("osd.%s" % proc[5]) + + try: + cname = self.get_all_containers_by_regex("ceph-osd*")[0][1] + except Exception: + cname = None + + self.add_cmd_output( + ["ceph daemon %s %s" % (i, c) for i in osd_ids for c in ceph_cmds], + container=cname + ) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/ceph.py sosreport-4.3/sos/report/plugins/ceph.py --- sosreport-4.2/sos/report/plugins/ceph.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ceph.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,141 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin -from socket import gethostname - - -class Ceph(Plugin, RedHatPlugin, UbuntuPlugin): - - short_desc = 'CEPH distributed storage' - - plugin_name = 'ceph' - profiles = ('storage', 'virt') - ceph_hostname = gethostname() - - packages = ( - 'ceph', - 'ceph-mds', - 'ceph-common', - 'libcephfs1', - 'ceph-fs-common', - 'calamari-server', - 'librados2' - ) - - services = ( - 'ceph-nfs@pacemaker', - 'ceph-mds@%s' % ceph_hostname, - 'ceph-mon@%s' % ceph_hostname, - 'ceph-mgr@%s' % ceph_hostname, - 'ceph-radosgw@*', - 'ceph-osd@*' - ) - - def setup(self): - all_logs = self.get_option("all_logs") - - self.add_file_tags({ - '.*/ceph.conf': 'ceph_conf', - '/var/log/ceph/ceph.log.*': 'ceph_log', - '/var/log/ceph/ceph-osd.*.log': 'ceph_osd_log' - }) - - if not all_logs: - self.add_copy_spec([ - "/var/log/ceph/*.log", - "/var/log/radosgw/*.log", - "/var/log/calamari/*.log" - ]) - else: - self.add_copy_spec([ - "/var/log/ceph/", - "/var/log/calamari", - "/var/log/radosgw" - ]) - - self.add_copy_spec([ - "/etc/ceph/", - "/etc/calamari/", - "/var/lib/ceph/", - "/run/ceph/" - ]) - - self.add_cmd_output([ - "ceph mon stat", - "ceph mon_status", - "ceph quorum_status", - "ceph mgr module ls", - "ceph mgr metadata", - "ceph balancer status", - "ceph osd metadata", - "ceph osd erasure-code-profile ls", - "ceph report", - "ceph osd crush show-tunables", - "ceph-disk list", - "ceph versions", - "ceph features", - "ceph insights", - "ceph osd crush dump", - "ceph -v", - "ceph-volume lvm list", - "ceph crash stat", - "ceph crash ls", - "ceph config log", - "ceph config generate-minimal-conf", - "ceph config-key dump", - ]) - - ceph_cmds = [ - "status", - "health detail", - "osd tree", - "osd stat", - "osd df tree", - "osd dump", - "osd df", - "osd perf", - "osd blocked-by", - "osd pool ls detail", - "osd pool autoscale-status", - "osd numa-status", - "device ls", - "mon dump", - "mgr dump", - "mds stat", - "df", - "df detail", - "fs ls", - "fs dump", - "pg dump", - "pg stat", - "time-sync-status", - ] - - self.add_cmd_output([ - "ceph %s" % s for s in ceph_cmds - ]) - - self.add_cmd_output([ - "ceph %s --format json-pretty" % s for s in ceph_cmds - ], subdir="json_output", tags="insights_ceph_health_detail") - - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/*keyring*", - "/var/lib/ceph/*/*keyring*", - "/var/lib/ceph/*/*/*keyring*", - "/var/lib/ceph/osd", - "/var/lib/ceph/mon", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) - -# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/ceph_rgw.py sosreport-4.3/sos/report/plugins/ceph_rgw.py --- sosreport-4.2/sos/report/plugins/ceph_rgw.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ceph_rgw.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,41 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +import glob + + +class CephRGW(Plugin, RedHatPlugin, UbuntuPlugin): + + short_desc = 'CEPH rgw' + + plugin_name = 'ceph_rgw' + profiles = ('storage', 'virt', 'container', 'webserver') + containers = ('ceph-rgw.*',) + + def check_enabled(self): + return True if glob.glob('/var/lib/ceph/radosgw/*/*') else False + + def setup(self): + self.add_copy_spec('/var/log/ceph/ceph-client.rgw*.log', + tags='ceph_rgw_log') + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/cman.py sosreport-4.3/sos/report/plugins/cman.py --- sosreport-4.2/sos/report/plugins/cman.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/cman.py 2022-02-15 04:20:20.000000000 +0000 @@ -21,9 +21,6 @@ files = ("/etc/cluster/cluster.conf",) - debugfs_path = "/sys/kernel/debug" - _debugfs_cleanup = False - def setup(self): self.add_copy_spec([ diff -Nru sosreport-4.2/sos/report/plugins/collectd.py sosreport-4.3/sos/report/plugins/collectd.py --- sosreport-4.2/sos/report/plugins/collectd.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/collectd.py 2022-02-15 04:20:20.000000000 +0000 @@ -33,7 +33,7 @@ p = re.compile('^LoadPlugin.*') try: - with open("/etc/collectd.conf") as f: + with open(self.path_join("/etc/collectd.conf"), 'r') as f: for line in f: if p.match(line): self.add_alert("Active Plugin found: %s" % diff -Nru sosreport-4.2/sos/report/plugins/conntrack.py sosreport-4.3/sos/report/plugins/conntrack.py --- sosreport-4.2/sos/report/plugins/conntrack.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/conntrack.py 2022-02-15 04:20:20.000000000 +0000 @@ -7,7 +7,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Conntrack(Plugin, IndependentPlugin): @@ -19,8 +19,8 @@ packages = ('conntrack-tools', 'conntrack', 'conntrackd') option_list = [ - ('namespaces', 'Number of namespaces to collect, 0 for unlimited', - 'slow', None) + PluginOpt("namespaces", default=None, val_type=int, + desc="Number of namespaces to collect, 0 for unlimited"), ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/container_log.py sosreport-4.3/sos/report/plugins/container_log.py --- sosreport-4.2/sos/report/plugins/container_log.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/container_log.py 2022-02-15 04:20:20.000000000 +0000 @@ -29,6 +29,6 @@ """Collect *.log files from subdirs of passed root path """ for dirName, _, _ in os.walk(root): - self.add_copy_spec(os.path.join(dirName, '*.log')) + self.add_copy_spec(self.path_join(dirName, '*.log')) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/containers_common.py sosreport-4.3/sos/report/plugins/containers_common.py --- sosreport-4.2/sos/report/plugins/containers_common.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/containers_common.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, PluginOpt import os @@ -19,8 +19,8 @@ profiles = ('container', ) packages = ('containers-common', ) option_list = [ - ('rootlessusers', 'colon-separated list of users\' containers info', - '', ''), + PluginOpt('rootlessusers', default='', val_type=str, + desc='colon-delimited list of users to collect for') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/convert2rhel.py sosreport-4.3/sos/report/plugins/convert2rhel.py --- sosreport-4.2/sos/report/plugins/convert2rhel.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/convert2rhel.py 2022-02-15 04:20:20.000000000 +0000 @@ -21,7 +21,8 @@ self.add_copy_spec([ "/var/log/convert2rhel/convert2rhel.log", - "/var/log/convert2rhel/rpm_va.log" + "/var/log/convert2rhel/archive/convert2rhel-*.log", + "/var/log/convert2rhel/rpm_va.log", ]) diff -Nru sosreport-4.2/sos/report/plugins/corosync.py sosreport-4.3/sos/report/plugins/corosync.py --- sosreport-4.2/sos/report/plugins/corosync.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/corosync.py 2022-02-15 04:20:20.000000000 +0000 @@ -47,7 +47,7 @@ # (it isnt precise but sufficient) pattern = r'^\s*(logging.)?logfile:\s*(\S+)$' try: - with open("/etc/corosync/corosync.conf") as f: + with open(self.path_join("/etc/corosync/corosync.conf"), 'r') as f: for line in f: if re.match(pattern, line): self.add_copy_spec(re.search(pattern, line).group(2)) diff -Nru sosreport-4.2/sos/report/plugins/crio.py sosreport-4.3/sos/report/plugins/crio.py --- sosreport-4.2/sos/report/plugins/crio.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/crio.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, SoSPredicate +from sos.report.plugins import (Plugin, RedHatPlugin, UbuntuPlugin, + SoSPredicate, PluginOpt) class CRIO(Plugin, RedHatPlugin, UbuntuPlugin): @@ -20,10 +21,10 @@ services = ('crio',) option_list = [ - ("all", "enable capture for all containers, even containers " - "that have terminated", 'fast', False), - ("logs", "capture logs for running containers", - 'fast', False), + PluginOpt('all', default=False, + desc='collect for all containers, even terminated ones'), + PluginOpt('logs', default=False, + desc='collect stdout/stderr logs for containers') ] def setup(self): @@ -77,11 +78,14 @@ images = self._get_crio_list(img_cmd) pods = self._get_crio_list(pod_cmd) + self._get_crio_goroutine_stacks() + for container in containers: - self.add_cmd_output("crictl inspect %s" % container) + self.add_cmd_output("crictl inspect %s" % container, + subdir="containers") if self.get_option('logs'): self.add_cmd_output("crictl logs -t %s" % container, - subdir="containers", priority=100) + subdir="containers/logs", priority=100) for image in images: self.add_cmd_output("crictl inspecti %s" % image, subdir="images") @@ -100,4 +104,13 @@ ret.pop(0) return ret + def _get_crio_goroutine_stacks(self): + result = self.exec_cmd("pidof crio") + if result['status'] != 0: + return + pid = result['output'].strip() + result = self.exec_cmd("kill -USR1 " + pid) + if result['status'] == 0: + self.add_copy_spec("/tmp/crio-goroutine-stacks*.log") + # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/dellrac.py sosreport-4.3/sos/report/plugins/dellrac.py --- sosreport-4.2/sos/report/plugins/dellrac.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/dellrac.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,49 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt + + +class DellRAC(Plugin, IndependentPlugin): + + short_desc = 'Dell Remote Access Controller Administration' + + plugin_name = 'dellrac' + profiles = ('system', 'storage', 'hardware',) + packages = ('srvadmin-idracadm7',) + + option_list = [ + PluginOpt('debug', default=False, desc='capture support assist data') + ] + + racadm = '/opt/dell/srvadmin/bin/idracadm7' + prefix = 'idracadm7' + + def setup(self): + for subcmd in ['getniccfg', 'getsysinfo']: + self.add_cmd_output( + '%s %s' % (self.racadm, subcmd), + suggest_filename='%s_%s' % (self.prefix, subcmd)) + + if self.get_option("debug"): + self.do_debug() + + def do_debug(self): + # ensure the sos_commands/dellrac directory does exist in either case + # as we will need to run the command at that dir, and also ensure + # logpath is properly populated in either case as well + try: + logpath = self.get_cmd_output_path() + except FileExistsError: + logpath = self.get_cmd_output_path(make=False) + subcmd = 'supportassist collect -f' + self.add_cmd_output( + '%s %s support.zip' % (self.racadm, subcmd), + runat=logpath, suggest_filename='%s_%s' % (self.prefix, subcmd)) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/discovery.py sosreport-4.3/sos/report/plugins/discovery.py --- sosreport-4.2/sos/report/plugins/discovery.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/discovery.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,31 @@ +# Copyright (C) 2021 Red Hat, Inc., Jose Castillo + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin + + +class Discovery(Plugin, RedHatPlugin): + + short_desc = 'Discovery inspection and reporting tool' + plugin_name = 'discovery' + packages = ('discovery', 'discovery-tools',) + + def setup(self): + self.add_copy_spec([ + "/root/discovery/db/volume/data/userdata/pg_log/", + "/root/discovery/server/volumes/log/app.log", + "/root/discovery/server/volumes/log/discovery-server.log" + ]) + + self.add_container_logs([ + 'discovery', + 'dsc-db' + ]) +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/dlm.py sosreport-4.3/sos/report/plugins/dlm.py --- sosreport-4.2/sos/report/plugins/dlm.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/dlm.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt import re @@ -18,7 +18,7 @@ profiles = ("cluster", ) packages = ("cman", "dlm", "pacemaker") option_list = [ - ("lockdump", "capture lock dumps for DLM", "slow", False), + PluginOpt('lockdump', default=False, desc='capture lock dumps for DLM') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/dmraid.py sosreport-4.3/sos/report/plugins/dmraid.py --- sosreport-4.2/sos/report/plugins/dmraid.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/dmraid.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Dmraid(Plugin, IndependentPlugin): @@ -18,7 +18,7 @@ packages = ('dmraid',) option_list = [ - ("metadata", "capture dmraid device metadata", "slow", False) + PluginOpt('metadata', default=False, desc='collect dmraid metadata') ] # V - {-V/--version} diff -Nru sosreport-4.2/sos/report/plugins/dnf.py sosreport-4.3/sos/report/plugins/dnf.py --- sosreport-4.2/sos/report/plugins/dnf.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/dnf.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class DNFPlugin(Plugin, RedHatPlugin): @@ -21,8 +21,10 @@ packages = ('dnf',) option_list = [ - ("history", "captures transaction history", "fast", False), - ("history-info", "detailed transaction history", "slow", False), + PluginOpt('history', default=False, + desc='collect transaction history'), + PluginOpt('history-info', default=False, + desc='collect detailed transaction history') ] def get_modules_info(self, modules): diff -Nru sosreport-4.2/sos/report/plugins/docker_distribution.py sosreport-4.3/sos/report/plugins/docker_distribution.py --- sosreport-4.2/sos/report/plugins/docker_distribution.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/docker_distribution.py 2022-02-15 04:20:20.000000000 +0000 @@ -19,8 +19,9 @@ def setup(self): self.add_copy_spec('/etc/docker-distribution/') self.add_journal('docker-distribution') - if self.path_exists('/etc/docker-distribution/registry/config.yml'): - with open('/etc/docker-distribution/registry/config.yml') as f: + conf = self.path_join('/etc/docker-distribution/registry/config.yml') + if self.path_exists(conf): + with open(conf) as f: for line in f: if 'rootdirectory' in line: loc = line.split()[1] diff -Nru sosreport-4.2/sos/report/plugins/docker.py sosreport-4.3/sos/report/plugins/docker.py --- sosreport-4.2/sos/report/plugins/docker.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/docker.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,7 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import (Plugin, RedHatPlugin, UbuntuPlugin, - SoSPredicate, CosPlugin) + SoSPredicate, CosPlugin, PluginOpt) class Docker(Plugin, CosPlugin): @@ -19,11 +19,12 @@ profiles = ('container',) option_list = [ - ("all", "enable capture for all containers, even containers " - "that have terminated", 'fast', False), - ("logs", "capture logs for running containers", - 'fast', False), - ("size", "capture image sizes for docker ps", 'slow', False) + PluginOpt('all', default=False, + desc='collect for all containers, even terminated ones'), + PluginOpt('logs', default=False, + desc='collect stdout/stderr logs for containers'), + PluginOpt('size', default=False, + desc='collect image sizes for docker ps') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/ds.py sosreport-4.3/sos/report/plugins/ds.py --- sosreport-4.2/sos/report/plugins/ds.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ds.py 2022-02-15 04:20:20.000000000 +0000 @@ -11,7 +11,6 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import Plugin, RedHatPlugin -import os class DirectoryServer(Plugin, RedHatPlugin): @@ -47,7 +46,7 @@ try: for d in self.listdir("/etc/dirsrv"): if d[0:5] == 'slapd': - certpath = os.path.join("/etc/dirsrv", d) + certpath = self.path_join("/etc/dirsrv", d) self.add_cmd_output("certutil -L -d %s" % certpath) self.add_cmd_output("dsctl %s healthcheck" % d) except OSError: diff -Nru sosreport-4.2/sos/report/plugins/ebpf.py sosreport-4.3/sos/report/plugins/ebpf.py --- sosreport-4.2/sos/report/plugins/ebpf.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ebpf.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt import json @@ -17,8 +17,8 @@ profiles = ('system', 'kernel', 'network') option_list = [ - ('namespaces', 'Number of namespaces to collect, 0 for unlimited', - 'slow', None) + PluginOpt("namespaces", default=None, val_type=int, + desc="Number of namespaces to collect, 0 for unlimited"), ] def get_bpftool_prog_ids(self, prog_json): diff -Nru sosreport-4.2/sos/report/plugins/elastic.py sosreport-4.3/sos/report/plugins/elastic.py --- sosreport-4.2/sos/report/plugins/elastic.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/elastic.py 2022-02-15 04:20:20.000000000 +0000 @@ -39,7 +39,9 @@ return hostname, port def setup(self): - els_config_file = "/etc/elasticsearch/elasticsearch.yml" + els_config_file = self.path_join( + "/etc/elasticsearch/elasticsearch.yml" + ) self.add_copy_spec(els_config_file) if self.get_option("all_logs"): diff -Nru sosreport-4.2/sos/report/plugins/etcd.py sosreport-4.3/sos/report/plugins/etcd.py --- sosreport-4.2/sos/report/plugins/etcd.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/etcd.py 2022-02-15 04:20:20.000000000 +0000 @@ -62,7 +62,7 @@ def get_etcd_url(self): try: - with open('/etc/etcd/etcd.conf', 'r') as ef: + with open(self.path_join('/etc/etcd/etcd.conf'), 'r') as ef: for line in ef: if line.startswith('ETCD_LISTEN_CLIENT_URLS'): return line.split('=')[1].replace('"', '').strip() diff -Nru sosreport-4.2/sos/report/plugins/fibrechannel.py sosreport-4.3/sos/report/plugins/fibrechannel.py --- sosreport-4.2/sos/report/plugins/fibrechannel.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/fibrechannel.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Fibrechannel(Plugin, RedHatPlugin): @@ -19,7 +19,7 @@ profiles = ('hardware', 'storage', 'system') files = ('/sys/class/fc_host', '/sys/class/fc_remote_ports') option_list = [ - ("debug", "enable debug logs", "fast", True) + PluginOpt('debug', default=True, desc='collect debugging logs') ] # vendor specific debug paths diff -Nru sosreport-4.2/sos/report/plugins/filesys.py sosreport-4.3/sos/report/plugins/filesys.py --- sosreport-4.2/sos/report/plugins/filesys.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/filesys.py 2022-02-15 04:20:20.000000000 +0000 @@ -7,10 +7,19 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, - UbuntuPlugin, CosPlugin) + UbuntuPlugin, CosPlugin, PluginOpt) class Filesys(Plugin, DebianPlugin, UbuntuPlugin, CosPlugin): + """Collects general information about the local filesystem(s) and mount + points as well as optional information about EXT filesystems. Note that + information specific filesystems such as XFS or ZFS is not collected by + this plugin, as there are specific plugins for those filesystem types. + + This plugin will collect /etc/fstab as well as mount information within + /proc/, and is responsible for the 'mount' and 'df' symlinks that appear + in an sos archive's root. + """ short_desc = 'Local file systems' @@ -18,9 +27,11 @@ profiles = ('storage',) option_list = [ - ("lsof", 'gathers information on all open files', 'slow', False), - ("dumpe2fs", 'dump filesystem information', 'slow', False), - ("frag", 'filesystem fragmentation status', 'slow', False) + PluginOpt('lsof', default=False, + desc='collect information on all open files'), + PluginOpt('dumpe2fs', default=False, desc='dump filesystem info'), + PluginOpt('frag', default=False, + desc='collect filesystem fragmentation status') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/firewalld.py sosreport-4.3/sos/report/plugins/firewalld.py --- sosreport-4.2/sos/report/plugins/firewalld.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/firewalld.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, SoSPredicate +from sos.report.plugins import Plugin, RedHatPlugin class FirewallD(Plugin, RedHatPlugin): @@ -35,12 +35,6 @@ "/var/log/firewalld", ]) - # collect nftables ruleset - nft_pred = SoSPredicate(self, - kmods=['nf_tables', 'nfnetlink'], - required={'kmods': 'all'}) - self.add_cmd_output("nft list ruleset", pred=nft_pred, changes=True) - # use a 10s timeout to workaround dbus problems in # docker containers. self.add_cmd_output([ diff -Nru sosreport-4.2/sos/report/plugins/firewall_tables.py sosreport-4.3/sos/report/plugins/firewall_tables.py --- sosreport-4.2/sos/report/plugins/firewall_tables.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/firewall_tables.py 2022-02-15 04:20:20.000000000 +0000 @@ -40,43 +40,61 @@ """ Collects nftables rulesets with 'nft' commands if the modules are present """ - self.add_cmd_output( - "nft list ruleset", - pred=SoSPredicate(self, kmods=['nf_tables']) - ) + # collect nftables ruleset + nft_pred = SoSPredicate(self, + kmods=['nf_tables', 'nfnetlink'], + required={'kmods': 'all'}) + return self.collect_cmd_output("nft list ruleset", pred=nft_pred, + changes=True) def setup(self): + # first, collect "nft list ruleset" as collecting commands like + # ip6tables -t mangle -nvL + # depends on its output + # store in nft_ip_tables lists of ip[|6] tables from nft list + nft_list = self.collect_nftables() + nft_ip_tables = {'ip': [], 'ip6': []} + nft_lines = nft_list['output'] if nft_list['status'] == 0 else '' + for line in nft_lines.splitlines(): + words = line.split()[0:3] + if len(words) == 3 and words[0] == 'table' and \ + words[1] in nft_ip_tables.keys(): + nft_ip_tables[words[1]].append(words[2]) # collect iptables -t for any existing table, if we can't read the # tables, collect 2 default ones (mangle, filter) + # do collect them only when relevant nft list ruleset exists + default_ip_tables = "mangle\nfilter\n" try: ip_tables_names = open("/proc/net/ip_tables_names").read() except IOError: - ip_tables_names = "mangle\nfilter\n" + ip_tables_names = default_ip_tables for table in ip_tables_names.splitlines(): - self.collect_iptable(table) + if nft_list['status'] == 0 and table in nft_ip_tables['ip']: + self.collect_iptable(table) # collect the same for ip6tables try: ip_tables_names = open("/proc/net/ip6_tables_names").read() except IOError: - ip_tables_names = "mangle\nfilter\n" + ip_tables_names = default_ip_tables for table in ip_tables_names.splitlines(): - self.collect_ip6table(table) + if nft_list['status'] == 0 and table in nft_ip_tables['ip6']: + self.collect_ip6table(table) - self.collect_nftables() - - # When iptables is called it will load the modules - # iptables_filter (for kernel <= 3) or - # nf_tables (for kernel >= 4) if they are not loaded. + # When iptables is called it will load: + # 1) the modules iptables_filter (for kernel <= 3) or + # nf_tables (for kernel >= 4) if they are not loaded. + # 2) nft 'ip filter' table will be created # The same goes for ipv6. - self.add_cmd_output( - "iptables -vnxL", - pred=SoSPredicate(self, kmods=['iptable_filter', 'nf_tables']) - ) - - self.add_cmd_output( - "ip6tables -vnxL", - pred=SoSPredicate(self, kmods=['ip6table_filter', 'nf_tables']) - ) + if nft_list['status'] != 0 or 'filter' in nft_ip_tables['ip']: + self.add_cmd_output( + "iptables -vnxL", + pred=SoSPredicate(self, kmods=['iptable_filter', 'nf_tables']) + ) + if nft_list['status'] != 0 or 'filter' in nft_ip_tables['ip6']: + self.add_cmd_output( + "ip6tables -vnxL", + pred=SoSPredicate(self, kmods=['ip6table_filter', 'nf_tables']) + ) self.add_copy_spec([ "/etc/nftables", diff -Nru sosreport-4.2/sos/report/plugins/foreman_installer.py sosreport-4.3/sos/report/plugins/foreman_installer.py --- sosreport-4.2/sos/report/plugins/foreman_installer.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/foreman_installer.py 2022-02-15 04:20:20.000000000 +0000 @@ -25,6 +25,8 @@ "/etc/foreman-installer/*", "/var/log/foreman-installer/*", "/var/log/foreman-maintain/*", + "/var/lib/foreman-maintain/data.yml", + "/etc/foreman-maintain/foreman_maintain.yml", # specifically collect .applied files # that would be skipped otherwise as hidden files "/etc/foreman-installer/scenarios.d/*/.applied", diff -Nru sosreport-4.2/sos/report/plugins/foreman_proxy.py sosreport-4.3/sos/report/plugins/foreman_proxy.py --- sosreport-4.2/sos/report/plugins/foreman_proxy.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/foreman_proxy.py 2022-02-15 04:20:20.000000000 +0000 @@ -38,6 +38,10 @@ self.apachepkg), "/var/log/{}*/katello-reverse-proxy_error_ssl.log*".format( self.apachepkg), + "/var/log/{}*/rhsm-pulpcore-https-*_access_ssl.log*".format( + self.apachepkg), + "/var/log/{}*/rhsm-pulpcore-https-*_error_ssl.log*".format( + self.apachepkg), ]) # collect http[|s]_proxy env.variables @@ -64,7 +68,7 @@ class DebianForemanProxy(ForemanProxy, DebianPlugin, UbuntuPlugin): - apachepkg = 'apache' + apachepkg = 'apache2' # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/foreman.py sosreport-4.3/sos/report/plugins/foreman.py --- sosreport-4.2/sos/report/plugins/foreman.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/foreman.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,10 +9,11 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, - UbuntuPlugin) +from sos.report.plugins import (Plugin, RedHatPlugin, SCLPlugin, + DebianPlugin, UbuntuPlugin, PluginOpt) from pipes import quote from re import match +from sos.utilities import is_executable class Foreman(Plugin): @@ -24,9 +25,14 @@ profiles = ('sysmgmt',) packages = ('foreman',) option_list = [ - ('months', 'number of months for dynflow output', 'fast', 1), - ('proxyfeatures', 'collect features of smart proxies', 'slow', False), + PluginOpt('months', default=1, + desc='number of months for dynflow output'), + PluginOpt('proxyfeatures', default=False, + desc='collect features of smart proxies'), + PluginOpt('puma-gc', default=False, + desc='collect Puma GC stats') ] + pumactl = 'pumactl %s -S /usr/share/foreman/tmp/puma.state' def setup(self): # for external DB, search in /etc/foreman/database.yml for: @@ -132,6 +138,17 @@ suggest_filename='dynflow_sidekiq_status') self.add_journal(units="dynflow-sidekiq@*") + # Puma stats & status, i.e. foreman-puma-stats, then + # pumactl stats -S /usr/share/foreman/tmp/puma.state + # and optionally also gc-stats + # if on RHEL with Software Collections, wrap the commands accordingly + if self.get_option('puma-gc'): + self.add_cmd_output(self.pumactl % 'gc-stats', + suggest_filename='pumactl_gc-stats') + self.add_cmd_output(self.pumactl % 'stats', + suggest_filename='pumactl_stats') + self.add_cmd_output('/usr/sbin/foreman-puma-status') + # collect tables sizes, ordered _cmd = self.build_query_cmd( "SELECT table_name, pg_size_pretty(total_bytes) AS total, " @@ -227,8 +244,16 @@ self.add_cmd_output(_cmd, suggest_filename=table, timeout=600, sizelimit=100, env=self.env) + # dynflow* tables on dynflow >=1.6.3 are encoded and hence in that + # case, psql-msgpack-decode wrapper tool from dynflow-utils (any + # version) must be used instead of plain psql command + dynutils = self.is_installed('dynflow-utils') for dyn in foremancsv: - _cmd = self.build_query_cmd(foremancsv[dyn], csv=True) + binary = "psql" + if dyn != 'foreman_tasks_tasks' and dynutils: + binary = "/usr/libexec/psql-msgpack-decode" + _cmd = self.build_query_cmd(foremancsv[dyn], csv=True, + binary=binary) self.add_cmd_output(_cmd, suggest_filename=dyn, timeout=600, sizelimit=100, env=self.env) @@ -253,7 +278,7 @@ # collect http[|s]_proxy env.variables self.add_env_var(["http_proxy", "https_proxy"]) - def build_query_cmd(self, query, csv=False): + def build_query_cmd(self, query, csv=False, binary="psql"): """ Builds the command needed to invoke the pgsql query as the postgres user. @@ -264,8 +289,8 @@ if csv: query = "COPY (%s) TO STDOUT " \ "WITH (FORMAT 'csv', DELIMITER ',', HEADER)" % query - _dbcmd = "psql --no-password -h %s -p 5432 -U foreman -d foreman -c %s" - return _dbcmd % (self.dbhost, quote(query)) + _dbcmd = "%s --no-password -h %s -p 5432 -U foreman -d foreman -c %s" + return _dbcmd % (binary, self.dbhost, quote(query)) def postproc(self): self.do_path_regex_sub( @@ -286,7 +311,7 @@ # attr so we can keep all log definitions centralized in the main class -class RedHatForeman(Foreman, RedHatPlugin): +class RedHatForeman(Foreman, SCLPlugin, RedHatPlugin): apachepkg = 'httpd' @@ -295,12 +320,18 @@ self.add_file_tags({ '/usr/share/foreman/.ssh/ssh_config': 'ssh_foreman_config', }) + # if we are on RHEL7 with scl, wrap some Puma commands by + # scl enable tfm 'command' + if self.policy.dist_version() == 7 and is_executable('scl'): + self.pumactl = "scl enable tfm '%s'" % self.pumactl super(RedHatForeman, self).setup() + self.add_cmd_output_scl('tfm', 'gem list', + suggest_filename='scl enable tfm gem list') class DebianForeman(Foreman, DebianPlugin, UbuntuPlugin): - apachepkg = 'apache' + apachepkg = 'apache2' # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/gcp.py sosreport-4.3/sos/report/plugins/gcp.py --- sosreport-4.2/sos/report/plugins/gcp.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/gcp.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,145 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. +import json +from http.client import HTTPResponse +from typing import Any +from urllib import request +from urllib.error import URLError + +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt + + +class GCP(Plugin, IndependentPlugin): + + short_desc = 'Google Cloud Platform' + plugin_name = 'gcp' + profiles = ('virt',) + + option_list = [ + PluginOpt('keep-pii', default=False, + desc="Stop the plugin from removing PIIs like project name " + "or organization ID from the metadata retrieved from " + "Metadata server.") + ] + + METADATA_ROOT = "http://metadata.google.internal/computeMetadata/v1/" + METADATA_QUERY = "http://metadata.google.internal/computeMetadata/v1/" \ + "?recursive=true" + REDACTED = "[--REDACTED--]" + + # A line we will be looking for in the dmesg output. If it's there, + # that means we're running on a Google Cloud Compute instance. + GOOGLE_DMI = "DMI: Google Google Compute Engine/Google " \ + "Compute Engine, BIOS Google" + + def check_enabled(self): + """ + Checks if this plugin should be executed at all. In this case, it + will check the `dmesg` command output to see if the system is + running on a Google Cloud Compute instance. + """ + dmesg = self.exec_cmd("dmesg") + if dmesg['status'] != 0: + return False + return self.GOOGLE_DMI in dmesg['output'] + + def setup(self): + """ + Collect the following info: + * Metadata from the Metadata server + * `gcloud auth list` output + * Any google services output from journal + """ + + # Capture gcloud auth list + self.add_cmd_output("gcloud auth list", tags=['gcp']) + + # Get and store Metadata + try: + self.metadata = self.get_metadata() + self.scrub_metadata() + self.add_string_as_file(json.dumps(self.metadata, indent=4), + "metadata.json", plug_dir=True, + tags=['gcp']) + except RuntimeError as err: + self.add_string_as_file(str(err), 'metadata.json', + plug_dir=True, tags=['gcp']) + + # Add journal entries + self.add_journal(units="google*", tags=['gcp']) + + def get_metadata(self) -> dict: + """ + Retrieves metadata from the Metadata Server and transforms it into a + dictionary object. + """ + response = self._query_address(self.METADATA_QUERY) + response_body = response.read().decode() + return json.loads(response_body) + + @staticmethod + def _query_address(url: str) -> HTTPResponse: + """ + Query the given url address with headers required by Google Metadata + Server. + """ + try: + req = request.Request(url, headers={'Metadata-Flavor': 'Google'}) + response = request.urlopen(req) + except URLError as err: + raise RuntimeError( + "Failed to communicate with Metadata Server: " + str(err)) + if response.code != 200: + raise RuntimeError( + f"Failed to communicate with Metadata Server " + f"(code: {response.code}): " + response.read().decode()) + return response + + def scrub_metadata(self): + """ + Remove all PII information from metadata, unless a keep-pii option + is specified. + + Note: PII information collected by this plugin, like + project number, account names etc. might be required by Google + Cloud Support for faster issue resolution. + """ + if self.get_option('keep-pii'): + return + + project_id = self.metadata['project']['projectId'] + project_number_int = self.metadata['project']['numericProjectId'] + project_number = str(project_number_int) + + def scrub(data: Any) -> Any: + if isinstance(data, dict): + if 'token' in data: + # Data returned for recursive query shouldn't contain + # tokens, but you can't be too careful. + data['token'] = self.REDACTED + return {scrub(k): scrub(v) for k, v in data.items()} + elif isinstance(data, list): + return [scrub(value) for value in data] + elif isinstance(data, str): + return data.replace(project_number, self.REDACTED)\ + .replace(project_id, self.REDACTED) + elif isinstance(data, int): + return self.REDACTED if data == project_number_int else data + return data + + self.metadata = scrub(self.metadata) + + self.safe_redact_key(self.metadata['project']['attributes'], + 'ssh-keys') + self.safe_redact_key(self.metadata['project']['attributes'], + 'sshKeys') + + @classmethod + def safe_redact_key(cls, dict_obj: dict, key: str): + if key in dict_obj: + dict_obj[key] = cls.REDACTED diff -Nru sosreport-4.2/sos/report/plugins/gfs2.py sosreport-4.3/sos/report/plugins/gfs2.py --- sosreport-4.2/sos/report/plugins/gfs2.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/gfs2.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Gfs2(Plugin, IndependentPlugin): @@ -18,9 +18,8 @@ packages = ("gfs2-utils",) option_list = [ - ("lockdump", - "capture lock dumps for all GFS2 filesystems", - "slow", False), + PluginOpt('lockdump', default=False, + desc='collect lock dumps for all GFS2 filesystems') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/gluster.py sosreport-4.3/sos/report/plugins/gluster.py --- sosreport-4.2/sos/report/plugins/gluster.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/gluster.py 2022-02-15 04:20:20.000000000 +0000 @@ -10,7 +10,7 @@ import os import glob import string -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Gluster(Plugin, RedHatPlugin): @@ -24,7 +24,9 @@ packages = ("glusterfs", "glusterfs-core") files = ("/etc/glusterd", "/var/lib/glusterd") - option_list = [("dump", "enable glusterdump support", "slow", False)] + option_list = [ + PluginOpt("dump", default=False, desc="enable glusterdump support") + ] def wait_for_statedump(self, name_dir): statedumps_present = 0 @@ -33,9 +35,10 @@ ] for statedump_file in statedump_entries: statedumps_present = statedumps_present+1 + _spath = self.path_join(name_dir, statedump_file) ret = -1 while ret == -1: - with open(name_dir + '/' + statedump_file, 'r') as sfile: + with open(_spath, 'r') as sfile: last_line = sfile.readlines()[-1] ret = string.count(last_line, 'DUMP_END_TIME') diff -Nru sosreport-4.2/sos/report/plugins/hpssm.py sosreport-4.3/sos/report/plugins/hpssm.py --- sosreport-4.2/sos/report/plugins/hpssm.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/hpssm.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,46 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt + + +class Hpssm(Plugin, IndependentPlugin): + + short_desc = 'HP Smart Storage Management' + + plugin_name = 'hpssm' + profiles = ('system', 'storage', 'hardware',) + packages = ('ilorest', 'ssacli', 'ssaducli',) + + option_list = [ + PluginOpt('debug', default=False, desc='capture debug data') + ] + + def setup(self): + self.add_cmd_output([ + 'ssacli ctrl slot=0 array all show detail', + 'ssacli ctrl slot=0 ld all show detail', + 'ssacli ctrl slot=0 pd all show detail', + 'ssacli ctrl slot=0 show detail', + ]) + + logpath = self.get_cmd_output_path() + + self.add_cmd_output( + 'ssaducli -v -adu -f %s/adu-log.zip' % logpath, + suggest_filename='ssaducli_-v_-adu.log') + + if self.get_option("debug"): + self.do_debug(logpath) + + def do_debug(self, logpath): + self.add_cmd_output( + 'ilorest serverlogs --selectlog=AHS', + runat=logpath, suggest_filename='ilorest.log') + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/__init__.py sosreport-4.3/sos/report/plugins/__init__.py --- sosreport-4.2/sos/report/plugins/__init__.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/__init__.py 2022-02-15 04:20:20.000000000 +0000 @@ -13,8 +13,9 @@ from sos.utilities import (sos_get_command_output, import_module, grep, fileobj, tail, is_executable, TIMEOUT_DEFAULT, path_exists, path_isdir, path_isfile, path_islink, - listdir) + listdir, path_join, bold) +from sos.archive import P_FILE import os import glob import re @@ -23,6 +24,7 @@ import logging import fnmatch import errno +import textwrap from datetime import datetime @@ -335,7 +337,7 @@ ] return " ".join(msg).lstrip() - def __nonzero__(self): + def __bool__(self): """Predicate evaluation hook. """ @@ -349,11 +351,6 @@ self._eval_arch()) and not self.dry_run) - def __bool__(self): - # Py3 evaluation ends in a __bool__() call where py2 ends in a call - # to __nonzero__(). Wrap the latter here, to support both versions - return self.__nonzero__() - def __init__(self, owner, dry_run=False, kmods=[], services=[], packages=[], cmd_outputs=[], arch=[], required={}): """Initialise a new SoSPredicate object @@ -406,7 +403,86 @@ sorted(self.__dict__.items())) -class Plugin(object): +class PluginOpt(): + """This is used to define options available to plugins. Plugins will need + to define options alongside their distro-specific classes in order to add + support for user-controlled changes in Plugin behavior. + + :param name: The name of the plugin option + :type name: ``str`` + + :param default: The default value of the option + :type default: Any + + :param desc: A short description of the effect of the option + :type desc: ``str`` + + :param long_desc: A detailed description of the option. Will be used by + `sos info` + :type long_desc: ``str`` + + :param val_type: The type of object the option accepts for values. If + not provided, auto-detect from the type of ``default`` + :type val_type: A single type or a ``list`` of types + """ + + name = '' + default = None + enabled = False + desc = '' + long_desc = '' + value = None + val_type = [None] + plugin = '' + + def __init__(self, name='undefined', default=None, desc='', long_desc='', + val_type=None): + self.name = name + self.default = default + self.desc = desc + self.long_desc = long_desc + self.value = self.default + if val_type is not None: + if not isinstance(val_type, list): + val_type = [val_type] + else: + val_type = [default.__class__] + self.val_type = val_type + + def __str__(self): + items = [ + 'name=%s' % self.name, + 'desc=\'%s\'' % self.desc, + 'value=%s' % self.value, + 'default=%s' % self.default + ] + return '(' + ', '.join(items) + ')' + + def __repr__(self): + return self.__str__() + + def set_value(self, val): + if not any([type(val) == _t for _t in self.val_type]): + valid = [] + for t in self.val_type: + if t is None: + continue + if t.__name__ == 'bool': + valid.append("boolean true/false (on/off, etc)") + elif t.__name__ == 'str': + valid.append("string (no spaces)") + elif t.__name__ == 'int': + valid.append("integer values") + raise Exception( + "Plugin option '%s.%s' takes %s, not %s" % ( + self.plugin, self.name, ', '.join(valid), + type(val).__name__ + ) + ) + self.value = val + + +class Plugin(): """This is the base class for sosreport plugins. Plugins should subclass this and set the class variables where applicable. @@ -474,12 +550,6 @@ # Default predicates predicate = None cmd_predicate = None - _default_plug_opts = [ - ('timeout', 'Timeout in seconds for plugin to finish', 'fast', -1), - ('cmd-timeout', 'Timeout in seconds for a command', 'fast', -1), - ('postproc', 'Enable post-processing collected plugin data', 'fast', - True) - ] def __init__(self, commons): @@ -488,13 +558,13 @@ self._env_vars = set() self.alerts = [] self.custom_text = "" - self.opt_names = [] - self.opt_parms = [] self.commons = commons self.forbidden_paths = [] self.copy_paths = set() + self.container_copy_paths = [] self.copy_strings = [] self.collect_cmds = [] + self.options = {} self.sysroot = commons['sysroot'] self.policy = commons['policy'] self.devices = commons['devices'] @@ -506,17 +576,32 @@ else logging.getLogger('sos') # add the default plugin opts - self.option_list.extend(self._default_plug_opts) - - # get the option list into a dictionary + self.options.update(self.get_default_plugin_opts()) + for popt in self.options: + self.options[popt].plugin = self.name() for opt in self.option_list: - self.opt_names.append(opt[0]) - self.opt_parms.append({'desc': opt[1], 'speed': opt[2], - 'enabled': opt[3]}) + opt.plugin = self.name() + self.options[opt.name] = opt # Initialise the default --dry-run predicate self.set_predicate(SoSPredicate(self)) + def get_default_plugin_opts(self): + return { + 'timeout': PluginOpt( + 'timeout', default=-1, val_type=int, + desc='Timeout in seconds for plugin to finish all collections' + ), + 'cmd-timeout': PluginOpt( + 'cmd-timeout', default=-1, val_type=int, + desc='Timeout in seconds for individual commands to finish' + ), + 'postproc': PluginOpt( + 'postproc', default=True, val_type=bool, + desc='Enable post-processing of collected data' + ) + } + def set_plugin_manifest(self, manifest): """Pass in a manifest object to the plugin to write to @@ -531,9 +616,13 @@ self.manifest.add_field('setup_start', '') self.manifest.add_field('setup_end', '') self.manifest.add_field('setup_time', '') + self.manifest.add_field('timeout', self.timeout) self.manifest.add_field('timeout_hit', False) + self.manifest.add_field('command_timeout', self.cmdtimeout) self.manifest.add_list('commands', []) self.manifest.add_list('files', []) + self.manifest.add_field('strings', {}) + self.manifest.add_field('containers', {}) def timeout_from_options(self, optname, plugoptname, default_timeout): """Returns either the default [plugin|cmd] timeout value, the value as @@ -613,6 +702,175 @@ return cls.plugin_name return cls.__name__.lower() + @classmethod + def display_help(cls, section): + if cls.plugin_name is None: + cls.display_self_help(section) + else: + cls.display_plugin_help(section) + + @classmethod + def display_plugin_help(cls, section): + from sos.help import TERMSIZE + section.set_title("%s Plugin Information - %s" + % (cls.plugin_name.title(), cls.short_desc)) + missing = '\nDetailed information is not available for this plugin.\n' + + # Concatenate the docstrings of distro-specific plugins with their + # base classes, if available. + try: + _doc = '' + _sc = cls.__mro__[1] + if _sc != Plugin and _sc.__doc__: + _doc = _sc.__doc__ + if cls.__doc__: + _doc += cls.__doc__ + except Exception: + _doc = None + + section.add_text('\n %s' % _doc if _doc else missing) + + if not any([cls.packages, cls.commands, cls.files, cls.kernel_mods, + cls.services, cls.containers]): + section.add_text("This plugin is always enabled by default.") + else: + for trig in ['packages', 'commands', 'files', 'kernel_mods', + 'services']: + if getattr(cls, trig, None): + section.add_text( + "Enabled by %s: %s" + % (trig, ', '.join(getattr(cls, trig))), + newline=False + ) + if getattr(cls, 'containers'): + section.add_text( + "Enabled by containers with names matching: %s" + % ', '.join(c for c in cls.containers), + newline=False + ) + + if cls.profiles: + section.add_text( + "Enabled with the following profiles: %s" + % ', '.join(p for p in cls.profiles), + newline=False + ) + + if hasattr(cls, 'verify_packages'): + section.add_text( + "\nVerfies packages (when using --verify): %s" + % ', '.join(pkg for pkg in cls.verify_packages), + newline=False, + ) + + if cls.postproc is not Plugin.postproc: + section.add_text( + 'This plugin performs post-processing on potentially ' + 'sensitive collections. Disabling post-processing may' + ' leave sensitive data in plaintext.' + ) + + if not cls.option_list: + return + + optsec = section.add_section('Plugin Options') + optsec.add_text( + "These options may be toggled or changed using '%s'" + % bold("-k %s.option_name=$value" % cls.plugin_name) + ) + optsec.add_text(bold( + "\n{:<4}{:<20}{:<30}{:<20}\n".format( + ' ', "Option Name", "Default", "Description") + ), newline=False + ) + + opt_indent = ' ' * 54 + for opt in cls.option_list: + _def = opt.default + # convert certain values to text meanings + if _def is None or _def == '': + _def = "None/Unset" + if isinstance(opt.default, bool): + if opt.default: + _def = "True/On" + else: + _def = "False/Off" + _ln = "{:<4}{:<20}{:<30}{:<20}".format(' ', opt.name, _def, + opt.desc) + optsec.add_text( + textwrap.fill(_ln, width=TERMSIZE, + subsequent_indent=opt_indent), + newline=False + ) + if opt.long_desc: + _size = TERMSIZE - 10 + space = ' ' * 8 + optsec.add_text( + textwrap.fill(opt.long_desc, width=_size, + initial_indent=space, + subsequent_indent=space), + newline=False + ) + + @classmethod + def display_self_help(cls, section): + section.set_title("SoS Plugin Detailed Help") + section.add_text( + "Plugins are what define what collections occur for a given %s " + "execution. Plugins are generally representative of a single " + "system component (e.g. kernel), package (e.g. podman), or similar" + " construct. Plugins will typically specify multiple files or " + "directories to copy, as well as commands to execute and collect " + "the output of for further analysis." + % bold('sos report') + ) + + subsec = section.add_section('Plugin Enablement') + subsec.add_text( + 'Plugins will be automatically enabled based on one of several ' + 'triggers - a certain package being installed, a command or file ' + 'existing, a kernel module being loaded, etc...' + ) + subsec.add_text( + "Plugins may also be enabled or disabled by name using the %s or " + "%s options respectively." + % (bold('-e $name'), bold('-n $name')) + ) + + subsec.add_text( + "Certain plugins may only be available for specific distributions " + "or may behave differently on different distributions based on how" + " the component for that plugin is installed or how it operates." + " When using %s, help will be displayed for the version of the " + "plugin appropriate for your distribution." + % bold('sos help report.plugins.$plugin') + ) + + optsec = section.add_section('Using Plugin Options') + optsec.add_text( + "Many plugins support additional options to enable/disable or in " + "some other way modify the collections it makes. Plugin options " + "are set using the %s syntax. Options that are on/off toggles " + "may exclude setting a value, which will be interpreted as " + "enabling that option.\n\nSee specific plugin help sections " + "or %s for more information on these options" + % (bold('-k $plugin_name.$option_name=$value'), + bold('sos report -l')) + ) + + seealso = section.add_section('See Also') + _also = { + 'report.plugins.$plugin': 'Help for a specific $plugin', + 'policies': 'Information on distribution policies' + } + seealso.add_text( + "Additional relevant information may be available in these " + "help sections:\n\n%s" % "\n".join( + "{:>8}{:<30}{:<30}".format(' ', sec, desc) + for sec, desc in _also.items() + ), newline=False + ) + def _format_msg(self, msg): return "[plugin:%s] %s" % (self.name(), msg) @@ -628,19 +886,6 @@ def _log_debug(self, msg): self.soslog.debug(self._format_msg(msg)) - def join_sysroot(self, path): - """Join a given path with the configured sysroot - - :param path: The filesystem path that needs to be joined - :type path: ``str`` - - :returns: The joined filesystem path - :rtype: ``str`` - """ - if path[0] == os.sep: - path = path[1:] - return os.path.join(self.sysroot, path) - def strip_sysroot(self, path): """Remove the configured sysroot from a filesystem path @@ -652,7 +897,7 @@ """ if not self.use_sysroot(): return path - if path.startswith(self.sysroot): + if self.sysroot and path.startswith(self.sysroot): return path[len(self.sysroot):] return path @@ -671,8 +916,10 @@ ``False`` :rtype: ``bool`` """ - paths = [self.sysroot, self.archive.get_tmp_dir()] - return os.path.commonprefix(paths) == self.sysroot + # if sysroot is still None, that implies '/' + _sysroot = self.sysroot or '/' + paths = [_sysroot, self.archive.get_tmp_dir()] + return os.path.commonprefix(paths) == _sysroot def is_installed(self, package_name): """Is the package $package_name installed? @@ -809,8 +1056,7 @@ return bool(pred) return False - def log_skipped_cmd(self, pred, cmd, kmods=False, services=False, - changes=False): + def log_skipped_cmd(self, cmd, pred, changes=False): """Log that a command was skipped due to predicate evaluation. Emit a warning message indicating that a command was skipped due @@ -820,21 +1066,17 @@ message indicating that the missing data can be collected by using the "--allow-system-changes" command line option will be included. - :param pred: The predicate that caused the command to be skipped - :type pred: ``SoSPredicate`` - :param cmd: The command that was skipped :type cmd: ``str`` - :param kmods: Did kernel modules cause the command to be skipped - :type kmods: ``bool`` - - :param services: Did services cause the command to be skipped - :type services: ``bool`` + :param pred: The predicate that caused the command to be skipped + :type pred: ``SoSPredicate`` :param changes: Is the `--allow-system-changes` enabled :type changes: ``bool`` """ + if pred is None: + pred = SoSPredicate(self) msg = "skipped command '%s': %s" % (cmd, pred.report_failure()) if changes: @@ -1096,7 +1338,7 @@ def _get_dest_for_srcpath(self, srcpath): if self.use_sysroot(): - srcpath = self.join_sysroot(srcpath) + srcpath = self.path_join(srcpath) for copied in self.copied_files: if srcpath == copied["srcpath"]: return copied["dstpath"] @@ -1204,17 +1446,13 @@ forbidden = [forbidden] if self.use_sysroot(): - forbidden = [self.join_sysroot(f) for f in forbidden] + forbidden = [self.path_join(f) for f in forbidden] for forbid in forbidden: self._log_info("adding forbidden path '%s'" % forbid) for path in glob.glob(forbid, recursive=recursive): self.forbidden_paths.append(path) - def get_all_options(self): - """return a list of all options selected""" - return (self.opt_names, self.opt_parms) - def set_option(self, optionname, value): """Set the named option to value. Ensure the original type of the option value is preserved @@ -1227,18 +1465,13 @@ :returns: ``True`` if the option is successfully set, else ``False`` :rtype: ``bool`` """ - for name, parms in zip(self.opt_names, self.opt_parms): - if name == optionname: - # FIXME: ensure that the resulting type of the set option - # matches that of the default value. This prevents a string - # option from being coerced to int simply because it holds - # a numeric value (e.g. a password). - # See PR #1526 and Issue #1597 - defaulttype = type(parms['enabled']) - if defaulttype != type(value) and defaulttype != type(None): - value = (defaulttype)(value) - parms['enabled'] = value + if optionname in self.options: + try: + self.options[optionname].set_value(value) return True + except Exception as err: + self._log_error(err) + raise return False def get_option(self, optionname, default=0): @@ -1266,29 +1499,12 @@ if optionname in global_options: return getattr(self.commons['cmdlineopts'], optionname) - for name, parms in zip(self.opt_names, self.opt_parms): - if name == optionname: - val = parms['enabled'] - if val is not None: - return val - else: - # if the value is `None`, use any non-zero default here, - # but still return `None` if no default is given since - # optionname did exist and had a `None` value - return default or val - - return default - - def get_option_as_list(self, optionname, delimiter=",", default=None): - """Will try to return the option as a list separated by the - delimiter. - """ - option = self.get_option(optionname) - try: - opt_list = [opt.strip() for opt in option.split(delimiter)] - return list(filter(None, opt_list)) - except Exception: + if optionname in self.options: + opt = self.options[optionname] + if not default or opt.value is not None: + return opt.value return default + return default def _add_copy_paths(self, copy_paths): self.copy_paths.update(copy_paths) @@ -1344,7 +1560,7 @@ self.manifest.files.append(manifest_data) def add_copy_spec(self, copyspecs, sizelimit=None, maxage=None, - tailit=True, pred=None, tags=[]): + tailit=True, pred=None, tags=[], container=None): """Add a file, directory, or regex matching filepaths to the archive :param copyspecs: A file, directory, or regex matching filepaths @@ -1369,10 +1585,17 @@ for this collection :type tags: ``str`` or a ``list`` of strings + :param container: Container(s) from which this file should be copied + :type container: ``str`` or a ``list`` of strings + `copyspecs` will be expanded and/or globbed as appropriate. Specifying a directory here will cause the plugin to attempt to collect the entire directory, recursively. + If `container` is specified, `copyspecs` may only be explicit paths, + not globs as currently container runtimes do not support glob expansion + as part of the copy operation. + Note that `sizelimit` is applied to each `copyspec`, not each file individually. For example, a copyspec of ``['/etc/foo', '/etc/bar.conf']`` and a `sizelimit` of 25 means that @@ -1384,7 +1607,7 @@ since = self.get_option('since') logarchive_pattern = re.compile(r'.*((\.(zip|gz|bz2|xz))|[-.][\d]+)$') - configfile_pattern = re.compile(r"^%s/*" % self.join_sysroot("etc")) + configfile_pattern = re.compile(r"^%s/*" % self.path_join("etc")) if not self.test_predicate(pred=pred): self._log_info("skipped copy spec '%s' due to predicate (%s)" % @@ -1409,28 +1632,79 @@ if isinstance(tags, str): tags = [tags] + def get_filename_tag(fname): + """Generate a tag to add for a single file copyspec + + This tag will be set to the filename, minus any extensions + except '.conf' which will be converted to '_conf' + """ + fname = fname.replace('-', '_') + if fname.endswith('.conf'): + return fname.replace('.', '_') + return fname.split('.')[0] + for copyspec in copyspecs: if not (copyspec and len(copyspec)): return False - if self.use_sysroot(): - copyspec = self.join_sysroot(copyspec) - - files = self._expand_copy_spec(copyspec) + if not container: + if self.use_sysroot(): + copyspec = self.path_join(copyspec) + files = self._expand_copy_spec(copyspec) + if len(files) == 0: + continue + else: + files = [copyspec] - if len(files) == 0: - continue + _spec_tags = [] + if len(files) == 1: + _spec_tags = [get_filename_tag(files[0].split('/')[-1])] - def get_filename_tag(fname): - """Generate a tag to add for a single file copyspec + _spec_tags.extend(tags) + _spec_tags = list(set(_spec_tags)) - This tag will be set to the filename, minus any extensions - except '.conf' which will be converted to '_conf' - """ - fname = fname.replace('-', '_') - if fname.endswith('.conf'): - return fname.replace('.', '_') - return fname.split('.')[0] + if container: + if isinstance(container, str): + container = [container] + for con in container: + if not self.container_exists(con): + continue + _tail = False + if sizelimit: + # to get just the size, stat requires a literal '%s' + # which conflicts with python string formatting + cmd = "stat -c %s " + copyspec + ret = self.exec_cmd(cmd, container=con) + if ret['status'] == 0: + try: + consize = int(ret['output']) + if consize > sizelimit: + _tail = True + except ValueError: + self._log_info( + "unable to determine size of '%s' in " + "container '%s'. Skipping collection." + % (copyspec, con) + ) + continue + else: + self._log_debug( + "stat of '%s' in container '%s' failed, " + "skipping collection: %s" + % (copyspec, con, ret['output']) + ) + continue + self.container_copy_paths.append( + (con, copyspec, sizelimit, _tail, _spec_tags) + ) + self._log_info( + "added collection of '%s' from container '%s'" + % (copyspec, con) + ) + # break out of the normal flow here as container file + # copies are done via command execution, not raw cp/mv + # operations + continue # Files hould be sorted in most-recently-modified order, so that # we collect the newest data first before reaching the limit. @@ -1454,12 +1728,6 @@ return False return True - _spec_tags = [] - if len(files) == 1: - _spec_tags = [get_filename_tag(files[0].split('/')[-1])] - - _spec_tags.extend(tags) - if since or maxage: files = list(filter(lambda f: time_filter(f), files)) @@ -1528,13 +1796,14 @@ # should collect the whole file and stop limit_reached = (sizelimit and current_size == sizelimit) - _spec_tags = list(set(_spec_tags)) - if self.manifest: - self.manifest.files.append({ - 'specification': copyspec, - 'files_copied': _manifest_files, - 'tags': _spec_tags - }) + if not container: + # container collection manifest additions are handled later + if self.manifest: + self.manifest.files.append({ + 'specification': copyspec, + 'files_copied': _manifest_files, + 'tags': _spec_tags + }) def add_blockdev_cmd(self, cmds, devices='block', timeout=None, sizelimit=None, chroot=True, runat=None, env=None, @@ -1629,7 +1898,7 @@ if not _dev_ok: continue if prepend_path: - device = os.path.join(prepend_path, device) + device = self.path_join(prepend_path, device) _cmd = cmd % {'dev': device} self._add_cmd_output(cmd=_cmd, timeout=timeout, sizelimit=sizelimit, chroot=chroot, @@ -1643,6 +1912,8 @@ kwargs['priority'] = 10 if 'changes' not in kwargs: kwargs['changes'] = False + if self.get_option('all_logs') or kwargs['sizelimit'] == 0: + kwargs['to_file'] = True soscmd = SoSCommand(**kwargs) self._log_debug("packed command: " + soscmd.__str__()) for _skip_cmd in self.skip_commands: @@ -1657,16 +1928,15 @@ self.collect_cmds.append(soscmd) self._log_info("added cmd output '%s'" % soscmd.cmd) else: - self.log_skipped_cmd(pred, soscmd.cmd, kmods=bool(pred.kmods), - services=bool(pred.services), - changes=soscmd.changes) + self.log_skipped_cmd(soscmd.cmd, pred, changes=soscmd.changes) def add_cmd_output(self, cmds, suggest_filename=None, root_symlink=None, timeout=None, stderr=True, chroot=True, runat=None, env=None, binary=False, sizelimit=None, pred=None, subdir=None, changes=False, foreground=False, tags=[], - priority=10, cmd_as_tag=False): + priority=10, cmd_as_tag=False, container=None, + to_file=False): """Run a program or a list of programs and collect the output Output will be limited to `sizelimit`, collecting the last X amount @@ -1731,6 +2001,14 @@ :param cmd_as_tag: Should the command string be automatically formatted to a tag? :type cmd_as_tag: ``bool`` + + :param container: Run the specified `cmds` inside a container with this + ID or name + :type container: ``str`` + + :param to_file: Should command output be written directly to a new + file rather than stored in memory? + :type to_file: ``bool`` """ if isinstance(cmds, str): cmds = [cmds] @@ -1741,13 +2019,24 @@ if pred is None: pred = self.get_predicate(cmd=True) for cmd in cmds: + container_cmd = None + if container: + ocmd = cmd + container_cmd = (ocmd, container) + cmd = self.fmt_container_cmd(container, cmd) + if not cmd: + self._log_debug("Skipping command '%s' as the requested " + "container '%s' does not exist." + % (ocmd, container)) + continue self._add_cmd_output(cmd=cmd, suggest_filename=suggest_filename, root_symlink=root_symlink, timeout=timeout, stderr=stderr, chroot=chroot, runat=runat, env=env, binary=binary, sizelimit=sizelimit, pred=pred, subdir=subdir, tags=tags, changes=changes, foreground=foreground, - priority=priority, cmd_as_tag=cmd_as_tag) + priority=priority, cmd_as_tag=cmd_as_tag, + to_file=to_file, container_cmd=container_cmd) def add_cmd_tags(self, tagdict): """Retroactively add tags to any commands that have been run by this @@ -1866,7 +2155,8 @@ # adds a mixed case variable name, still get that as well self._env_vars.update([env, env.upper(), env.lower()]) - def add_string_as_file(self, content, filename, pred=None): + def add_string_as_file(self, content, filename, pred=None, plug_dir=False, + tags=[]): """Add a string to the archive as a file :param content: The string to write to the archive @@ -1878,6 +2168,14 @@ :param pred: A predicate to gate if the string should be added to the archive or not :type pred: ``SoSPredicate`` + + :param plug_dir: Should the string be saved under the plugin's dir in + sos_commands/? If false, save to sos_strings/ + :type plug_dir: ``bool` + + :param tags: A tag or set of tags to add to the manifest entry for this + collection + :type tags: ``str`` or a ``list`` of strings """ # Generate summary string for logging @@ -1890,7 +2188,13 @@ (summary, self.get_predicate(pred=pred))) return - self.copy_strings.append((content, filename)) + sos_dir = 'sos_commands' if plug_dir else 'sos_strings' + filename = os.path.join(sos_dir, self.name(), filename) + + if isinstance(tags, str): + tags = [tags] + + self.copy_strings.append((content, filename, tags)) self._log_debug("added string ...'%s' as '%s'" % (summary, filename)) def _collect_cmd_output(self, cmd, suggest_filename=None, @@ -1898,7 +2202,8 @@ stderr=True, chroot=True, runat=None, env=None, binary=False, sizelimit=None, subdir=None, changes=False, foreground=False, tags=[], - priority=10, cmd_as_tag=False): + priority=10, cmd_as_tag=False, to_file=False, + container_cmd=False): """Execute a command and save the output to a file for inclusion in the report. @@ -1920,8 +2225,12 @@ :param subdir: Subdir in plugin directory to save to :param changes: Does this cmd potentially make a change on the system? + :param foreground: Run the `cmd` in the foreground with a + TTY :param tags: Add tags in the archive manifest :param cmd_as_tag: Format command string to tag + :param to_file: Write output directly to file instead + of saving in memory :returns: dict containing status, output, and filename in the archive for the executed cmd @@ -1951,27 +2260,46 @@ else: root = None + if suggest_filename: + outfn = self._make_command_filename(suggest_filename, subdir) + else: + outfn = self._make_command_filename(cmd, subdir) + + outfn_strip = outfn[len(self.commons['cmddir'])+1:] + + if to_file: + self._log_debug("collecting '%s' output directly to disk" + % cmd) + self.archive.check_path(outfn, P_FILE) + out_file = os.path.join(self.archive.get_archive_path(), outfn) + else: + out_file = False + start = time() result = sos_get_command_output( cmd, timeout=timeout, stderr=stderr, chroot=root, chdir=runat, env=env, binary=binary, sizelimit=sizelimit, - poller=self.check_timeout, foreground=foreground + poller=self.check_timeout, foreground=foreground, + to_file=out_file ) end = time() run_time = end - start if result['status'] == 124: - self._log_warn( - "command '%s' timed out after %ds" % (cmd, timeout) - ) + warn = "command '%s' timed out after %ds" % (cmd, timeout) + self._log_warn(warn) + if to_file: + msg = (" - output up until the timeout may be available at " + "%s" % outfn) + self._log_debug("%s%s" % (warn, msg)) manifest_cmd = { 'command': cmd.split(' ')[0], 'parameters': cmd.split(' ')[1:], 'exec': cmd, - 'filepath': None, + 'filepath': outfn if to_file else None, 'truncated': result['truncated'], 'return_code': result['status'], 'priority': priority, @@ -1992,7 +2320,7 @@ result = sos_get_command_output( cmd, timeout=timeout, chroot=False, chdir=runat, env=env, binary=binary, sizelimit=sizelimit, - poller=self.check_timeout + poller=self.check_timeout, to_file=out_file ) run_time = time() - start self._log_debug("could not run '%s': command not found" % cmd) @@ -2009,22 +2337,15 @@ if result['truncated']: self._log_info("collected output of '%s' was truncated" % cmd.split()[0]) - - if suggest_filename: - outfn = self._make_command_filename(suggest_filename, subdir) - else: - outfn = self._make_command_filename(cmd, subdir) - - outfn_strip = outfn[len(self.commons['cmddir'])+1:] - - if result['truncated']: linkfn = outfn outfn = outfn.replace('sos_commands', 'sos_strings') + '.tailed' - if binary: - self.archive.add_binary(result['output'], outfn) - else: - self.archive.add_string(result['output'], outfn) + if not to_file: + if binary: + self.archive.add_binary(result['output'], outfn) + else: + self.archive.add_string(result['output'], outfn) + if result['truncated']: # we need to manually build the relative path from the paths that # exist within the build dir to properly drop these symlinks @@ -2044,17 +2365,22 @@ os.path.join(self.archive.get_archive_path(), outfn) if outfn else '' ) + if self.manifest: manifest_cmd['filepath'] = outfn manifest_cmd['run_time'] = run_time self.manifest.commands.append(manifest_cmd) + if container_cmd: + self._add_container_cmd_to_manifest(manifest_cmd.copy(), + container_cmd) return result def collect_cmd_output(self, cmd, suggest_filename=None, root_symlink=False, timeout=None, stderr=True, chroot=True, runat=None, env=None, binary=False, sizelimit=None, pred=None, - subdir=None, tags=[]): + changes=False, foreground=False, subdir=None, + tags=[]): """Execute a command and save the output to a file for inclusion in the report, then return the results for further use by the plugin @@ -2097,6 +2423,9 @@ on the system? :type changes: ``bool`` + :param foreground: Run the `cmd` in the foreground with a TTY + :type foreground: ``bool`` + :param tags: Add tags in the archive manifest :type tags: ``str`` or a ``list`` of strings @@ -2105,8 +2434,7 @@ :rtype: ``dict`` """ if not self.test_predicate(cmd=True, pred=pred): - self._log_info("skipped cmd output '%s' due to predicate (%s)" % - (cmd, self.get_predicate(cmd=True, pred=pred))) + self.log_skipped_cmd(cmd, pred, changes=changes) return { 'status': None, # don't match on if result['status'] checks 'output': '', @@ -2116,8 +2444,8 @@ return self._collect_cmd_output( cmd, suggest_filename=suggest_filename, root_symlink=root_symlink, timeout=timeout, stderr=stderr, chroot=chroot, runat=runat, - env=env, binary=binary, sizelimit=sizelimit, subdir=subdir, - tags=tags + env=env, binary=binary, sizelimit=sizelimit, foreground=foreground, + subdir=subdir, tags=tags ) def exec_cmd(self, cmd, timeout=None, stderr=True, chroot=True, @@ -2194,6 +2522,60 @@ chdir=runat, binary=binary, env=env, foreground=foreground, stderr=stderr) + def _add_container_file_to_manifest(self, container, path, arcpath, tags): + """Adds a file collection to the manifest for a particular container + and file path. + + :param container: The name of the container + :type container: ``str`` + + :param path: The filename from the container filesystem + :type path: ``str`` + + :param arcpath: Where in the archive the file is written to + :type arcpath: ``str`` + + :param tags: Metadata tags for this collection + :type tags: ``str`` or ``list`` of strings + """ + if container not in self.manifest.containers: + self.manifest.containers[container] = {'files': [], 'commands': []} + self.manifest.containers[container]['files'].append({ + 'specification': path, + 'files_copied': arcpath, + 'tags': tags + }) + + def _add_container_cmd_to_manifest(self, manifest, contup): + """Adds a command collection to the manifest for a particular container + and creates a symlink to that collection from the relevant + sos_containers/ location + + :param manifest: The manifest entry for the command + :type manifest: ``dict`` + + :param contup: A tuple of (original_cmd, container_name) + :type contup: ``tuple`` + """ + + cmd, container = contup + if container not in self.manifest.containers: + self.manifest.containers[container] = {'files': [], 'commands': []} + manifest['exec'] = cmd + manifest['command'] = cmd.split(' ')[0] + manifest['parameters'] = cmd.split(' ')[1:] + + _cdir = "sos_containers/%s/sos_commands/%s" % (container, self.name()) + _outloc = "../../../../%s" % manifest['filepath'] + cmdfn = self._mangle_command(cmd) + conlnk = "%s/%s" % (_cdir, cmdfn) + + self.archive.check_path(conlnk, P_FILE) + os.symlink(_outloc, self.archive.dest_path(conlnk)) + + manifest['filepath'] = conlnk + self.manifest.containers[container]['commands'].append(manifest) + def _get_container_runtime(self, runtime=None): """Based on policy and request by the plugin, return a usable ContainerRuntime if one exists @@ -2318,20 +2700,32 @@ return _runtime.volumes return [] - def get_container_logs(self, container, **kwargs): - """Helper to get the ``logs`` output for a given container + def add_container_logs(self, containers, get_all=False, **kwargs): + """Helper to get the ``logs`` output for a given container or list + of container names and/or regexes. Supports passthru of add_cmd_output() options - :param container: The name of the container to retrieve logs from - :type container: ``str`` + :param containers: The name of the container to retrieve logs from, + may be a single name or a regex + :type containers: ``str`` or ``list` of strs + + :param get_all: Should non-running containers also be queried? + Default: False + :type get_all: ``bool`` :param kwargs: Any kwargs supported by ``add_cmd_output()`` are supported here """ _runtime = self._get_container_runtime() if _runtime is not None: - self.add_cmd_output(_runtime.get_logs_command(container), **kwargs) + if isinstance(containers, str): + containers = [containers] + for container in containers: + _cons = self.get_all_containers_by_regex(container, get_all) + for _con in _cons: + cmd = _runtime.get_logs_command(_con[1]) + self.add_cmd_output(cmd, **kwargs) def fmt_container_cmd(self, container, cmd, quotecmd=False): """Format a command to be executed by the loaded ``ContainerRuntime`` @@ -2353,7 +2747,7 @@ if self.container_exists(container): _runtime = self._get_container_runtime() return _runtime.fmt_container_cmd(container, cmd, quotecmd) - return cmd + return '' def is_module_loaded(self, module_name): """Determine whether specified module is loaded or not @@ -2391,7 +2785,7 @@ :param services: Service name(s) to collect statuses for :type services: ``str`` or a ``list`` of strings - :param kwargs: Optional arguments to pass to _add_cmd_output + :param kwargs: Optional arguments to pass to add_cmd_output (timeout, predicate, suggest_filename,..) """ @@ -2406,7 +2800,7 @@ return for service in services: - self._add_cmd_output(cmd="%s %s" % (query, service), **kwargs) + self.add_cmd_output("%s %s" % (query, service), **kwargs) def add_journal(self, units=None, boot=None, since=None, until=None, lines=None, allfields=False, output=None, @@ -2464,6 +2858,9 @@ all_logs = self.get_option("all_logs") log_size = sizelimit or self.get_option("log_size") log_size = max(log_size, journal_size) if not all_logs else 0 + if sizelimit == 0: + # allow for specific sizelimit overrides in plugins + log_size = 0 if isinstance(units, str): units = [units] @@ -2521,7 +2918,7 @@ if self.path_isfile(path) or self.path_islink(path): found_paths.append(path) elif self.path_isdir(path) and self.listdir(path): - found_paths.extend(__expand(os.path.join(path, '*'))) + found_paths.extend(__expand(self.path_join(path, '*'))) else: found_paths.append(path) except PermissionError: @@ -2537,7 +2934,7 @@ if (os.access(copyspec, os.R_OK) and self.path_isdir(copyspec) and self.listdir(copyspec)): # the directory exists and is non-empty, recurse through it - copyspec = os.path.join(copyspec, '*') + copyspec = self.path_join(copyspec, '*') expanded = glob.glob(copyspec, recursive=True) recursed_files = [] for _path in expanded: @@ -2556,11 +2953,53 @@ return list(set(expanded)) def _collect_copy_specs(self): - for path in self.copy_paths: + for path in sorted(self.copy_paths, reverse=True): self._log_info("collecting path '%s'" % path) self._do_copy_path(path) self.generate_copyspec_tags() + def _collect_container_copy_specs(self): + """Copy any requested files from containers here. This is done + separately from normal file collection as this requires the use of + a container runtime. + + This method will iterate over self.container_copy_paths which is a set + of 5-tuples as (container, path, sizelimit, stdout, tags). + """ + if not self.container_copy_paths: + return + rt = self._get_container_runtime() + if not rt: + self._log_info("Cannot collect container based files - no runtime " + "is present/active.") + return + if not rt.check_can_copy(): + self._log_info("Loaded runtime '%s' does not support copying " + "files from containers. Skipping collections.") + return + for contup in self.container_copy_paths: + con, path, sizelimit, tailit, tags = contup + self._log_info("collecting '%s' from container '%s'" % (path, con)) + + arcdest = "sos_containers/%s/%s" % (con, path.lstrip('/')) + self.archive.check_path(arcdest, P_FILE) + dest = self.archive.dest_path(arcdest) + + cpcmd = rt.get_copy_command( + con, path, dest, sizelimit=sizelimit if tailit else None + ) + cpret = self.exec_cmd(cpcmd, timeout=10) + + if cpret['status'] == 0: + if tailit: + # current runtimes convert files sent to stdout to tar + # archives, with no way to control that + self.archive.add_string(cpret['output'], arcdest) + self._add_container_file_to_manifest(con, path, arcdest, tags) + else: + self._log_info("error copying '%s' from container '%s': %s" + % (path, con, cpret['output'])) + def _collect_cmds(self): self.collect_cmds.sort(key=lambda x: x.priority) for soscmd in self.collect_cmds: @@ -2569,7 +3008,7 @@ self._collect_cmd_output(**soscmd.__dict__) def _collect_strings(self): - for string, file_name in self.copy_strings: + for string, file_name, tags in self.copy_strings: if self._timeout_hit: return content = '' @@ -2580,10 +3019,12 @@ self._log_info("collecting string ...'%s' as '%s'" % (content, file_name)) try: - self.archive.add_string(string, - os.path.join('sos_strings', - self.name(), - file_name)) + self.archive.add_string(string, file_name) + _name = file_name.split('/')[-1].replace('.', '_') + self.manifest.strings[_name] = { + 'path': file_name, + 'tags': tags + } except Exception as e: self._log_debug("could not add string '%s': %s" % (file_name, e)) @@ -2592,6 +3033,7 @@ """Collect the data for a plugin.""" start = time() self._collect_copy_specs() + self._collect_container_copy_specs() self._collect_cmds() self._collect_strings() fields = (self.name(), time() - start) @@ -2658,8 +3100,8 @@ # SCL containers don't exist ()): type(self)._scls_matched.append(scl) - if type(self)._scls_matched: - return True + if type(self)._scls_matched: + return True return self._check_plugin_triggers(self.files, self.packages, @@ -2682,7 +3124,7 @@ return ((any(self.path_exists(fname) for fname in files) or any(self.is_installed(pkg) for pkg in packages) or - any(is_executable(cmd) for cmd in commands) or + any(is_executable(cmd, self.sysroot) for cmd in commands) or any(self.is_module_loaded(mod) for mod in self.kernel_mods) or any(self.is_service(svc) for svc in services) or any(self.container_exists(cntr) for cntr in containers)) and @@ -2750,7 +3192,7 @@ :returns: True if the path exists in sysroot, else False :rtype: ``bool`` """ - return path_exists(path, self.commons['cmdlineopts'].sysroot) + return path_exists(path, self.sysroot) def path_isdir(self, path): """Helper to call the sos.utilities wrapper that allows the @@ -2763,7 +3205,7 @@ :returns: True if the path is a dir, else False :rtype: ``bool`` """ - return path_isdir(path, self.commons['cmdlineopts'].sysroot) + return path_isdir(path, self.sysroot) def path_isfile(self, path): """Helper to call the sos.utilities wrapper that allows the @@ -2776,7 +3218,7 @@ :returns: True if the path is a file, else False :rtype: ``bool`` """ - return path_isfile(path, self.commons['cmdlineopts'].sysroot) + return path_isfile(path, self.sysroot) def path_islink(self, path): """Helper to call the sos.utilities wrapper that allows the @@ -2789,7 +3231,7 @@ :returns: True if the path is a link, else False :rtype: ``bool`` """ - return path_islink(path, self.commons['cmdlineopts'].sysroot) + return path_islink(path, self.sysroot) def listdir(self, path): """Helper to call the sos.utilities wrapper that allows the @@ -2802,7 +3244,21 @@ :returns: Contents of path, if it is a directory :rtype: ``list`` """ - return listdir(path, self.commons['cmdlineopts'].sysroot) + return listdir(path, self.sysroot) + + def path_join(self, path, *p): + """Helper to call the sos.utilities wrapper that allows the + corresponding `os` call to account for sysroot + + :param path: The leading path passed to os.path.join() + :type path: ``str`` + + :param p: Following path section(s) to be joined with ``path``, + an empty parameter will result in a path that ends with + a separator + :type p: ``str`` + """ + return path_join(path, *p, sysroot=self.sysroot) def postproc(self): """Perform any postprocessing. To be replaced by a plugin if required. @@ -2823,7 +3279,7 @@ try: cmd_line_paths = glob.glob(cmd_line_glob) for path in cmd_line_paths: - f = open(path, 'r') + f = open(self.path_join(path), 'r') cmd_line = f.read().strip() if process in cmd_line: status = True @@ -2872,21 +3328,20 @@ ) for ns in ns_list: # if ns_pattern defined, skip namespaces not matching the pattern - if ns_pattern: - if not bool(re.match(pattern, ns)): - continue + if ns_pattern and not bool(re.match(pattern, ns)): + continue + out_ns.append(ns) - # if ns_max is defined at all, limit returned list to that number + # if ns_max is defined at all, break the loop when the limit is + # reached # this allows the use of both '0' and `None` to mean unlimited - elif ns_max: - out_ns.append(ns) + if ns_max: if len(out_ns) == ns_max: self._log_warn("Limiting namespace iteration " "to first %s namespaces found" % ns_max) break - else: - out_ns.append(ns) + return out_ns @@ -2930,24 +3385,9 @@ return [scl.strip() for scl in output.splitlines()] def convert_cmd_scl(self, scl, cmd): - """wrapping command in "scl enable" call and adds proper PATH + """wrapping command in "scl enable" call """ - # load default SCL prefix to PATH - prefix = self.policy.get_default_scl_prefix() - # read prefix from /etc/scl/prefixes/${scl} and strip trailing '\n' - try: - prefix = open('/etc/scl/prefixes/%s' % scl, 'r').read()\ - .rstrip('\n') - except Exception as e: - self._log_error("Failed to find prefix for SCL %s using %s: %s" - % (scl, prefix, e)) - - # expand PATH by equivalent prefixes under the SCL tree - path = os.environ["PATH"] - for p in path.split(':'): - path = '%s/%s%s:%s' % (prefix, scl, p, path) - - scl_cmd = "scl enable %s \"PATH=%s %s\"" % (scl, path, cmd) + scl_cmd = "scl enable %s \"%s\"" % (scl, cmd) return scl_cmd def add_cmd_output_scl(self, scl, cmds, **kwargs): @@ -3005,6 +3445,11 @@ pass +class OpenEulerPlugin(object): + """Tagging class for openEuler linux distributions""" + pass + + class CosPlugin(object): """Tagging class for Container-Optimized OS""" pass diff -Nru sosreport-4.2/sos/report/plugins/insights.py sosreport-4.3/sos/report/plugins/insights.py --- sosreport-4.2/sos/report/plugins/insights.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/insights.py 2022-02-15 04:20:20.000000000 +0000 @@ -32,6 +32,11 @@ else: self.add_copy_spec("/var/log/insights-client/insights-client.log") + self.add_cmd_output( + "insights-client --test-connection --net-debug", + timeout=30 + ) + def postproc(self): for conf in self.config: self.do_file_sub( diff -Nru sosreport-4.2/sos/report/plugins/ipmitool.py sosreport-4.3/sos/report/plugins/ipmitool.py --- sosreport-4.2/sos/report/plugins/ipmitool.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ipmitool.py 2022-02-15 04:20:20.000000000 +0000 @@ -26,12 +26,14 @@ if result['status'] == 0: cmd += " -I usb" + for subcmd in ['channel info', 'channel getaccess', 'lan print']: + for channel in [1, 3]: + self.add_cmd_output("%s %s %d" % (cmd, subcmd, channel)) + # raw 0x30 0x65: Get HDD drive Fault LED State # raw 0x30 0xb0: Get LED Status self.add_cmd_output([ - "%s channel info 3" % cmd, - "%s channel getaccess 3" % cmd, "%s raw 0x30 0x65" % cmd, "%s raw 0x30 0xb0" % cmd, "%s sel info" % cmd, @@ -40,7 +42,6 @@ "%s sensor list" % cmd, "%s chassis status" % cmd, "%s lan print" % cmd, - "%s lan print 3" % cmd, "%s fru print" % cmd, "%s mc info" % cmd, "%s sdr info" % cmd diff -Nru sosreport-4.2/sos/report/plugins/jars.py sosreport-4.3/sos/report/plugins/jars.py --- sosreport-4.2/sos/report/plugins/jars.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/jars.py 2022-02-15 04:20:20.000000000 +0000 @@ -14,7 +14,7 @@ import re import zipfile from functools import partial -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Jars(Plugin, RedHatPlugin): @@ -24,9 +24,10 @@ plugin_name = "jars" profiles = ("java",) option_list = [ - ("append_locations", "colon-separated list of additional JAR paths", - "fast", ""), - ("all_known_locations", "scan all known paths", "slow", False) + PluginOpt('append_locations', default="", val_type=str, + desc='colon-delimited list of additional JAR paths'), + PluginOpt('all_known_locations', default=False, + desc='scan all known paths') ] # There is no standard location for JAR files and scanning @@ -62,7 +63,7 @@ for location in locations: for dirpath, _, filenames in os.walk(location): for filename in filenames: - path = os.path.join(dirpath, filename) + path = self.path_join(dirpath, filename) if Jars.is_jar(path): jar_paths.append(path) @@ -78,7 +79,7 @@ results["jars"].append(record) results_str = json.dumps(results, indent=4, separators=(",", ": ")) - self.add_string_as_file(results_str, "jars.json") + self.add_string_as_file(results_str, "jars.json", plug_dir=True) @staticmethod def is_jar(path): diff -Nru sosreport-4.2/sos/report/plugins/kdump.py sosreport-4.3/sos/report/plugins/kdump.py --- sosreport-4.2/sos/report/plugins/kdump.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/kdump.py 2022-02-15 04:20:20.000000000 +0000 @@ -40,7 +40,7 @@ packages = ('kexec-tools',) def fstab_parse_fs(self, device): - with open('/etc/fstab', 'r') as fp: + with open(self.path_join('/etc/fstab'), 'r') as fp: for line in fp: if line.startswith((device)): return line.split()[1].rstrip('/') @@ -50,7 +50,7 @@ fs = "" path = "/var/crash" - with open('/etc/kdump.conf', 'r') as fp: + with open(self.path_join('/etc/kdump.conf'), 'r') as fp: for line in fp: if line.startswith("path"): path = line.split()[1] diff -Nru sosreport-4.2/sos/report/plugins/kernel.py sosreport-4.3/sos/report/plugins/kernel.py --- sosreport-4.2/sos/report/plugins/kernel.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/kernel.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,11 +6,25 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt import glob class Kernel(Plugin, IndependentPlugin): + """The Kernel plugin is aimed at collecting general information about + the locally running kernel. This information should be distribution-neutral + using commands and filesystem collections that are ubiquitous across + distributions. + + Debugging information from /sys/kernel/debug is collected by default, + however care is taken so that these collections avoid areas like + /sys/kernel/debug/tracing/trace_pipe which would otherwise cause the + sos collection attempt to appear to 'hang'. + + The 'trace' option will enable the collection of the + /sys/kernel/debug/tracing/trace file specfically, but will not change the + behavior stated above otherwise. + """ short_desc = 'Linux kernel' @@ -21,8 +35,10 @@ sys_module = '/sys/module' option_list = [ - ("with-timer", "gather /proc/timer* statistics", "slow", False), - ("trace", "gather /sys/kernel/debug/tracing/trace file", "slow", False) + PluginOpt('with-timer', default=False, + desc='gather /proc/timer* statistics'), + PluginOpt('trace', default=False, + desc='gather /sys/kernel/debug/tracing/trace file') ] def setup(self): @@ -110,7 +126,8 @@ "/sys/kernel/debug/extfrag/unusable_index", "/sys/kernel/debug/extfrag/extfrag_index", clocksource_path + "available_clocksource", - clocksource_path + "current_clocksource" + clocksource_path + "current_clocksource", + "/proc/pressure/" ]) if self.get_option("with-timer"): diff -Nru sosreport-4.2/sos/report/plugins/kubernetes.py sosreport-4.3/sos/report/plugins/kubernetes.py --- sosreport-4.2/sos/report/plugins/kubernetes.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/kubernetes.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, PluginOpt from fnmatch import translate import re @@ -22,13 +22,14 @@ profiles = ('container',) option_list = [ - ("all", "also collect all namespaces output separately", - 'slow', False), - ("describe", "capture descriptions of all kube resources", - 'fast', False), - ("podlogs", "capture logs for pods", 'slow', False), - ("podlogs-filter", "only capture logs for pods matching this string", - 'fast', '') + PluginOpt('all', default=False, + desc='collect all namespace output separately'), + PluginOpt('describe', default=False, + desc='collect describe output of all resources'), + PluginOpt('podlogs', default=False, + desc='capture stdout/stderr logs from pods'), + PluginOpt('podlogs-filter', default='', val_type=str, + desc='only collect logs from pods matching this pattern') ] kube_cmd = "kubectl" diff -Nru sosreport-4.2/sos/report/plugins/libraries.py sosreport-4.3/sos/report/plugins/libraries.py --- sosreport-4.2/sos/report/plugins/libraries.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/libraries.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Libraries(Plugin, IndependentPlugin): @@ -17,7 +17,8 @@ profiles = ('system',) option_list = [ - ('ldconfigv', 'collect verbose ldconfig output', "slow", False) + PluginOpt('ldconfigv', default=False, + desc='collect verbose ldconfig output') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/libreswan.py sosreport-4.3/sos/report/plugins/libreswan.py --- sosreport-4.2/sos/report/plugins/libreswan.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/libreswan.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin, SoSPredicate +from sos.report.plugins import (Plugin, IndependentPlugin, SoSPredicate, + PluginOpt) class Libreswan(Plugin, IndependentPlugin): @@ -19,8 +20,8 @@ plugin_name = 'libreswan' profiles = ('network', 'security', 'openshift') option_list = [ - ("ipsec-barf", "collect the output of the ipsec barf command", - "slow", False) + PluginOpt('ipsec-barf', default=False, + desc='collect ipsec barf output') ] files = ('/etc/ipsec.conf',) diff -Nru sosreport-4.2/sos/report/plugins/libvirt.py sosreport-4.3/sos/report/plugins/libvirt.py --- sosreport-4.2/sos/report/plugins/libvirt.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/libvirt.py 2022-02-15 04:20:20.000000000 +0000 @@ -50,12 +50,17 @@ "/var/log/libvirt/libvirtd.log", "/var/log/libvirt/qemu/*.log*", "/var/log/libvirt/lxc/*.log", - "/var/log/libvirt/uml/*.log" + "/var/log/libvirt/uml/*.log", + "/var/log/containers/libvirt/libvirtd.log", + "/var/log/containers/libvirt/qemu/*.log*", + "/var/log/containers/libvirt/lxc/*.log", + "/var/log/containers/libvirt/uml/*.log", ]) else: self.add_copy_spec("/var/log/libvirt") + self.add_copy_spec("/var/log/containers/libvirt") - if self.path_exists(self.join_sysroot(libvirt_keytab)): + if self.path_exists(self.path_join(libvirt_keytab)): self.add_cmd_output("klist -ket %s" % libvirt_keytab) self.add_cmd_output("ls -lR /var/lib/libvirt/qemu") diff -Nru sosreport-4.2/sos/report/plugins/logs.py sosreport-4.3/sos/report/plugins/logs.py --- sosreport-4.2/sos/report/plugins/logs.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/logs.py 2022-02-15 04:20:20.000000000 +0000 @@ -24,15 +24,15 @@ since = self.get_option("since") if self.path_exists('/etc/rsyslog.conf'): - with open('/etc/rsyslog.conf', 'r') as conf: + with open(self.path_join('/etc/rsyslog.conf'), 'r') as conf: for line in conf.readlines(): if line.startswith('$IncludeConfig'): confs += glob.glob(line.split()[1]) for conf in confs: - if not self.path_exists(conf): + if not self.path_exists(self.path_join(conf)): continue - config = self.join_sysroot(conf) + config = self.path_join(conf) logs += self.do_regex_find_all(r"^\S+\s+(-?\/.*$)\s+", config) for i in logs: @@ -60,7 +60,7 @@ # - there is some data present, either persistent or runtime only # - systemd-journald service exists # otherwise fallback to collecting few well known logfiles directly - journal = any([self.path_exists(p + "/log/journal/") + journal = any([self.path_exists(self.path_join(p, "log/journal/")) for p in ["/var", "/run"]]) if journal and self.is_service("systemd-journald"): self.add_journal(since=since, tags='journal_full', priority=100) diff -Nru sosreport-4.2/sos/report/plugins/lvm2.py sosreport-4.3/sos/report/plugins/lvm2.py --- sosreport-4.2/sos/report/plugins/lvm2.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/lvm2.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin, SoSPredicate +from sos.report.plugins import (Plugin, IndependentPlugin, SoSPredicate, + PluginOpt) class Lvm2(Plugin, IndependentPlugin): @@ -16,10 +17,12 @@ plugin_name = 'lvm2' profiles = ('storage',) - option_list = [("lvmdump", 'collect an lvmdump tarball', 'fast', False), - ("lvmdump-am", 'attempt to collect an lvmdump with ' - 'advanced options and raw metadata collection', 'slow', - False)] + option_list = [ + PluginOpt('lvmdump', default=False, desc='collect an lvmdump tarball'), + PluginOpt('lvmdump-am', default=False, + desc=('attempt to collect lvmdump with advanced options and ' + 'raw metadata')) + ] def do_lvmdump(self, metadata=False): """Collects an lvmdump in standard format with optional metadata diff -Nru sosreport-4.2/sos/report/plugins/maas.py sosreport-4.3/sos/report/plugins/maas.py --- sosreport-4.2/sos/report/plugins/maas.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/maas.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, UbuntuPlugin +from sos.report.plugins import Plugin, UbuntuPlugin, PluginOpt class Maas(Plugin, UbuntuPlugin): @@ -33,11 +33,12 @@ ) option_list = [ - ('profile-name', - 'The name with which you will later refer to this remote', '', ''), - ('url', 'The URL of the remote API', '', ''), - ('credentials', - 'The credentials, also known as the API key', '', '') + PluginOpt('profile-name', default='', val_type=str, + desc='Name of the remote API'), + PluginOpt('url', default='', val_type=str, + desc='URL of the remote API'), + PluginOpt('credentials', default='', val_type=str, + desc='Credentials, or the API key') ] def _has_login_options(self): diff -Nru sosreport-4.2/sos/report/plugins/manageiq.py sosreport-4.3/sos/report/plugins/manageiq.py --- sosreport-4.2/sos/report/plugins/manageiq.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/manageiq.py 2022-02-15 04:20:20.000000000 +0000 @@ -58,7 +58,7 @@ # Log files to collect from miq_dir/log/ miq_log_dir = os.path.join(miq_dir, "log") - miq_main_log_files = [ + miq_main_logs = [ 'ansible_tower.log', 'top_output.log', 'evm.log', @@ -81,16 +81,16 @@ self.add_copy_spec(list(self.files)) self.add_copy_spec([ - os.path.join(self.miq_conf_dir, x) for x in self.miq_conf_files + self.path_join(self.miq_conf_dir, x) for x in self.miq_conf_files ]) # Collect main log files without size limit. self.add_copy_spec([ - os.path.join(self.miq_log_dir, x) for x in self.miq_main_log_files + self.path_join(self.miq_log_dir, x) for x in self.miq_main_logs ], sizelimit=0) self.add_copy_spec([ - os.path.join(self.miq_log_dir, x) for x in self.miq_log_files + self.path_join(self.miq_log_dir, x) for x in self.miq_log_files ]) self.add_copy_spec([ @@ -101,8 +101,8 @@ if environ.get("APPLIANCE_PG_DATA"): pg_dir = environ.get("APPLIANCE_PG_DATA") self.add_copy_spec([ - os.path.join(pg_dir, 'pg_log'), - os.path.join(pg_dir, 'postgresql.conf') + self.path_join(pg_dir, 'pg_log'), + self.path_join(pg_dir, 'postgresql.conf') ]) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/monit.py sosreport-4.3/sos/report/plugins/monit.py --- sosreport-4.2/sos/report/plugins/monit.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/monit.py 2022-02-15 04:20:20.000000000 +0000 @@ -29,8 +29,6 @@ # Define log files monit_log = ["/var/log/monit.log"] - option_list = [] - def setup(self): self.add_cmd_output("monit status") self.add_copy_spec(self.monit_log + self.monit_conf) diff -Nru sosreport-4.2/sos/report/plugins/mssql.py sosreport-4.3/sos/report/plugins/mssql.py --- sosreport-4.2/sos/report/plugins/mssql.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/mssql.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class MsSQL(Plugin, RedHatPlugin): @@ -20,8 +20,8 @@ packages = ('mssql-server',) option_list = [ - ('mssql_conf', 'SQL Server configuration file.', '', - '/var/opt/mssql/mssql.conf') + PluginOpt('mssql_conf', default='/var/opt/mssql/mssql.conf', + desc='SQL server configuration file') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/mysql.py sosreport-4.3/sos/report/plugins/mysql.py --- sosreport-4.2/sos/report/plugins/mysql.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/mysql.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) import os @@ -21,9 +22,11 @@ pw_warn_text = " (password visible in process listings)" option_list = [ - ("dbuser", "username for database dumps", "", "mysql"), - ("dbpass", "password for database dumps" + pw_warn_text, "", ""), - ("dbdump", "collect a database dump", "", False) + PluginOpt('dbuser', default='mysql', val_type=str, + desc='username for database dump collection'), + PluginOpt('dbpass', default='', val_type=str, + desc='password for data dump collection' + pw_warn_text), + PluginOpt('dbdump', default=False, desc='Collect a database dump') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/navicli.py sosreport-4.3/sos/report/plugins/navicli.py --- sosreport-4.2/sos/report/plugins/navicli.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/navicli.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt from sos.utilities import is_executable @@ -19,8 +19,10 @@ plugin_name = 'navicli' profiles = ('storage', 'hardware') - option_list = [("ipaddrs", "list of space separated CLARiiON IP addresses", - '', "")] + option_list = [ + PluginOpt('ipaddrs', default='', val_type=str, + desc='space-delimited list of CLARiiON IP addresses') + ] def check_enabled(self): return is_executable("navicli") diff -Nru sosreport-4.2/sos/report/plugins/networking.py sosreport-4.3/sos/report/plugins/networking.py --- sosreport-4.2/sos/report/plugins/networking.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/networking.py 2022-02-15 04:20:20.000000000 +0000 @@ -7,7 +7,7 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import (Plugin, RedHatPlugin, UbuntuPlugin, - DebianPlugin, SoSPredicate) + DebianPlugin, SoSPredicate, PluginOpt) class Networking(Plugin): @@ -17,17 +17,20 @@ plugin_name = "networking" profiles = ('network', 'hardware', 'system') trace_host = "www.example.com" + option_list = [ - ("traceroute", "collect a traceroute to %s" % trace_host, "slow", - False), - ("namespace_pattern", "Specific namespaces pattern to be " + - "collected, namespaces pattern should be separated by whitespace " + - "as for example \"eth* ens2\"", "fast", ""), - ("namespaces", "Number of namespaces to collect, 0 for unlimited. " + - "Incompatible with the namespace_pattern option", "slow", None), - ("ethtool_namespaces", "Define if ethtool commands should be " + - "collected for namespaces", "slow", True), - ("eepromdump", "collect 'ethtool -e' for all devices", "slow", False) + PluginOpt("traceroute", default=False, + desc="collect a traceroute to %s" % trace_host), + PluginOpt("namespace_pattern", default="", val_type=str, + desc=("Specific namespace names or patterns to collect, " + "whitespace delimited.")), + PluginOpt("namespaces", default=None, val_type=int, + desc="Number of namespaces to collect, 0 for unlimited"), + PluginOpt("ethtool_namespaces", default=True, + desc=("Toggle if ethtool commands should be run for each " + "namespace")), + PluginOpt("eepromdump", default=False, + desc="Toggle collection of 'ethtool -e' for NICs") ] # switch to enable netstat "wide" (non-truncated) output mode @@ -39,7 +42,6 @@ def setup(self): super(Networking, self).setup() - for opt in self.ethtool_shortopts: self.add_cmd_tags({ 'ethtool -%s .*' % opt: 'ethool_%s' % opt @@ -93,6 +95,8 @@ "networkctl status -a", "ip route show table all", "ip -6 route show table all", + "ip -d route show cache", + "ip -d -6 route show cache", "ip -4 rule", "ip -6 rule", "ip -s -d link", @@ -110,6 +114,7 @@ self.add_cmd_output([ "devlink dev param show", "devlink dev info", + "devlink port show", ]) devlinks = self.collect_cmd_output("devlink dev") @@ -129,7 +134,7 @@ ss_cmd = "ss -peaonmi" ss_pred = SoSPredicate(self, kmods=[ 'tcp_diag', 'udp_diag', 'inet_diag', 'unix_diag', 'netlink_diag', - 'af_packet_diag' + 'af_packet_diag', 'xsk_diag' ], required={'kmods': 'all'}) self.add_cmd_output(ss_cmd, pred=ss_pred, changes=True) @@ -180,38 +185,56 @@ # per-namespace. self.add_cmd_output("ip netns") cmd_prefix = "ip netns exec " - for namespace in self.get_network_namespaces( - self.get_option("namespace_pattern"), - self.get_option("namespaces")): + namespaces = self.get_network_namespaces( + self.get_option("namespace_pattern"), + self.get_option("namespaces")) + if (namespaces): + # 'ip netns exec iptables-save' must be guarded by nf_tables + # kmod, if 'iptables -V' output contains 'nf_tables' + # analogously for ip6tables + co = {'cmd': 'iptables -V', 'output': 'nf_tables'} + co6 = {'cmd': 'ip6tables -V', 'output': 'nf_tables'} + iptables_with_nft = (SoSPredicate(self, kmods=['nf_tables']) + if self.test_predicate(self, + pred=SoSPredicate(self, cmd_outputs=co)) + else None) + ip6tables_with_nft = (SoSPredicate(self, kmods=['nf_tables']) + if self.test_predicate(self, + pred=SoSPredicate(self, cmd_outputs=co6)) + else None) + for namespace in namespaces: + _subdir = "namespaces/%s" % namespace ns_cmd_prefix = cmd_prefix + namespace + " " self.add_cmd_output([ - ns_cmd_prefix + "ip address show", + ns_cmd_prefix + "ip -d address show", ns_cmd_prefix + "ip route show table all", ns_cmd_prefix + "ip -s -s neigh show", ns_cmd_prefix + "ip rule list", - ns_cmd_prefix + "iptables-save", - ns_cmd_prefix + "ip6tables-save", ns_cmd_prefix + "netstat %s -neopa" % self.ns_wide, ns_cmd_prefix + "netstat -s", ns_cmd_prefix + "netstat %s -agn" % self.ns_wide, ns_cmd_prefix + "nstat -zas", - ], priority=50) + ], priority=50, subdir=_subdir) + self.add_cmd_output([ns_cmd_prefix + "iptables-save"], + pred=iptables_with_nft, + subdir=_subdir, + priority=50) + self.add_cmd_output([ns_cmd_prefix + "ip6tables-save"], + pred=ip6tables_with_nft, + subdir=_subdir, + priority=50) ss_cmd = ns_cmd_prefix + "ss -peaonmi" # --allow-system-changes is handled directly in predicate # evaluation, so plugin code does not need to separately # check for it - self.add_cmd_output(ss_cmd, pred=ss_pred) + self.add_cmd_output(ss_cmd, pred=ss_pred, subdir=_subdir) - # Collect ethtool commands only when ethtool_namespaces - # is set to true. - if self.get_option("ethtool_namespaces"): - # Devices that exist in a namespace use less ethtool - # parameters. Run this per namespace. - for namespace in self.get_network_namespaces( - self.get_option("namespace_pattern"), - self.get_option("namespaces")): - ns_cmd_prefix = cmd_prefix + namespace + " " + # Collect ethtool commands only when ethtool_namespaces + # is set to true. + if self.get_option("ethtool_namespaces"): + # Devices that exist in a namespace use less ethtool + # parameters. Run this per namespace. netns_netdev_list = self.exec_cmd( ns_cmd_prefix + "ls -1 /sys/class/net/" ) @@ -226,9 +249,7 @@ ns_cmd_prefix + "ethtool -i " + eth, ns_cmd_prefix + "ethtool -k " + eth, ns_cmd_prefix + "ethtool -S " + eth - ], priority=50) - - return + ], priority=50, subdir=_subdir) class RedHatNetworking(Networking, RedHatPlugin): diff -Nru sosreport-4.2/sos/report/plugins/networkmanager.py sosreport-4.3/sos/report/plugins/networkmanager.py --- sosreport-4.2/sos/report/plugins/networkmanager.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/networkmanager.py 2022-02-15 04:20:20.000000000 +0000 @@ -25,6 +25,8 @@ "/etc/NetworkManager/dispatcher.d" ]) + self.add_journal(units="NetworkManager") + # There are some incompatible changes in nmcli since # the release of NetworkManager >= 0.9.9. In addition, # NetworkManager >= 0.9.9 will use the long names of diff -Nru sosreport-4.2/sos/report/plugins/nginx.py sosreport-4.3/sos/report/plugins/nginx.py --- sosreport-4.2/sos/report/plugins/nginx.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/nginx.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Nginx(Plugin, IndependentPlugin): @@ -17,7 +17,7 @@ packages = ('nginx',) option_list = [ - ("log", "gathers all nginx logs", "slow", False) + PluginOpt('log', default=False, desc='collect all nginx logs') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/npm.py sosreport-4.3/sos/report/plugins/npm.py --- sosreport-4.2/sos/report/plugins/npm.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/npm.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,7 @@ # See the LICENSE file in the source distribution for further information. import os -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Npm(Plugin, IndependentPlugin): @@ -17,10 +17,10 @@ short_desc = 'Information from available npm modules' plugin_name = 'npm' profiles = ('system',) - option_list = [("project_path", - 'List npm modules of a project specified by path', - 'fast', - '')] + option_list = [ + PluginOpt('project_path', default='', val_type=str, + desc='Collect npm modules of project at this path') + ] # in Fedora, Debian, Ubuntu and Suse the package is called npm packages = ('npm',) diff -Nru sosreport-4.2/sos/report/plugins/numa.py sosreport-4.3/sos/report/plugins/numa.py --- sosreport-4.2/sos/report/plugins/numa.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/numa.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,6 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import Plugin, IndependentPlugin -import os.path class Numa(Plugin, IndependentPlugin): @@ -42,10 +41,10 @@ ]) self.add_copy_spec([ - os.path.join(numa_path, "node*/meminfo"), - os.path.join(numa_path, "node*/cpulist"), - os.path.join(numa_path, "node*/distance"), - os.path.join(numa_path, "node*/hugepages/hugepages-*/*") + self.path_join(numa_path, "node*/meminfo"), + self.path_join(numa_path, "node*/cpulist"), + self.path_join(numa_path, "node*/distance"), + self.path_join(numa_path, "node*/hugepages/hugepages-*/*") ]) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/nvidia.py sosreport-4.3/sos/report/plugins/nvidia.py --- sosreport-4.2/sos/report/plugins/nvidia.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/nvidia.py 2022-02-15 04:20:20.000000000 +0000 @@ -23,13 +23,24 @@ '--list-gpus', '-q -d PERFORMANCE', '-q -d SUPPORTED_CLOCKS', - '-q -d PAGE_RETIREMENT' + '-q -d PAGE_RETIREMENT', + '-q', + '-q -d ECC', + 'nvlink -s', + 'nvlink -e' ] self.add_cmd_output(["nvidia-smi %s" % cmd for cmd in subcmds]) query = ('gpu_name,gpu_bus_id,vbios_version,temperature.gpu,' - 'utilization.gpu,memory.total,memory.free,memory.used') + 'utilization.gpu,memory.total,memory.free,memory.used,' + 'clocks.applications.graphics,clocks.applications.memory') + querypages = ('timestamp,gpu_bus_id,gpu_serial,gpu_uuid,' + 'retired_pages.address,retired_pages.cause') self.add_cmd_output("nvidia-smi --query-gpu=%s --format=csv" % query) + self.add_cmd_output( + "nvidia-smi --query-retired-pages=%s --format=csv" % querypages + ) + self.add_journal(boot=0, identifier='nvidia-persistenced') # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/nvme.py sosreport-4.3/sos/report/plugins/nvme.py --- sosreport-4.2/sos/report/plugins/nvme.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/nvme.py 2022-02-15 04:20:20.000000000 +0000 @@ -26,6 +26,7 @@ cmds = [ "smartctl --all %(dev)s", + "smartctl --all %(dev)s -j", "nvme list-ns %(dev)s", "nvme fw-log %(dev)s", "nvme list-ctrl %(dev)s", diff -Nru sosreport-4.2/sos/report/plugins/omnipath_client.py sosreport-4.3/sos/report/plugins/omnipath_client.py --- sosreport-4.2/sos/report/plugins/omnipath_client.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/omnipath_client.py 2022-02-15 04:20:20.000000000 +0000 @@ -45,7 +45,12 @@ # rather than storing it somewhere under /var/tmp and copying it via # add_copy_spec, add it directly to sos_commands/ dir by # building a path argument using self.get_cmd_output_path(). - self.add_cmd_output("opacapture %s" % join(self.get_cmd_output_path(), - "opacapture.tgz")) + # This command calls 'depmod -a', so lets make sure we + # specified the 'allow-system-changes' option before running it. + if self.get_option('allow_system_changes'): + self.add_cmd_output("opacapture %s" % + join(self.get_cmd_output_path(), + "opacapture.tgz"), + changes=True) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/opencontrail.py sosreport-4.3/sos/report/plugins/opencontrail.py --- sosreport-4.2/sos/report/plugins/opencontrail.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/opencontrail.py 2022-02-15 04:20:20.000000000 +0000 @@ -25,8 +25,7 @@ cnames = self.get_containers(get_all=True) cnames = [c[1] for c in cnames if 'opencontrail' in c[1]] for cntr in cnames: - _cmd = self.fmt_container_cmd(cntr, 'contrail-status') - self.add_cmd_output(_cmd) + self.add_cmd_output('contrail-status', container=cntr) else: self.add_cmd_output("contrail-status") diff -Nru sosreport-4.2/sos/report/plugins/openshift_ovn.py sosreport-4.3/sos/report/plugins/openshift_ovn.py --- sosreport-4.2/sos/report/plugins/openshift_ovn.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openshift_ovn.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,41 @@ +# Copyright (C) 2021 Nadia Pinaeva + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin + + +class OpenshiftOVN(Plugin, RedHatPlugin): + """This plugin is used to collect OCP 4.x OVN logs. + """ + short_desc = 'Openshift OVN' + plugin_name = "openshift_ovn" + containers = ('ovnkube-master', 'ovn-ipsec') + profiles = ('openshift',) + + def setup(self): + self.add_copy_spec([ + "/var/lib/ovn/etc/ovnnb_db.db", + "/var/lib/ovn/etc/ovnsb_db.db", + "/var/lib/openvswitch/etc/keys", + "/var/log/openvswitch/libreswan.log", + "/var/log/openvswitch/ovs-monitor-ipsec.log" + ]) + + self.add_cmd_output([ + 'ovn-appctl -t /var/run/ovn/ovnnb_db.ctl ' + + 'cluster/status OVN_Northbound', + 'ovn-appctl -t /var/run/ovn/ovnsb_db.ctl ' + + 'cluster/status OVN_Southbound'], + container='ovnkube-master') + self.add_cmd_output([ + 'ovs-appctl -t ovs-monitor-ipsec tunnels/show', + 'ipsec status', + 'certutil -L -d sql:/etc/ipsec.d'], + container='ovn-ipsec') diff -Nru sosreport-4.2/sos/report/plugins/openshift.py sosreport-4.3/sos/report/plugins/openshift.py --- sosreport-4.2/sos/report/plugins/openshift.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openshift.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt from fnmatch import translate import os import re @@ -54,17 +54,19 @@ packages = ('openshift-hyperkube',) option_list = [ - ('token', 'admin token to allow API queries', 'fast', None), - ('host', 'host address to use for oc login, including port', 'fast', - 'https://localhost:6443'), - ('no-oc', 'do not collect `oc` command output', 'fast', False), - ('podlogs', 'collect logs from each pod', 'fast', True), - ('podlogs-filter', ('limit podlogs collection to pods matching this ' - 'regex'), 'fast', ''), - ('only-namespaces', 'colon-delimited list of namespaces to collect', - 'fast', ''), - ('add-namespaces', ('colon-delimited list of namespaces to add to the ' - 'default collections'), 'fast', '') + PluginOpt('token', default=None, val_type=str, + desc='admin token to allow API queries'), + PluginOpt('host', default='https://localhost:6443', + desc='host address to use for oc login, including port'), + PluginOpt('no-oc', default=False, desc='do not collect `oc` output'), + PluginOpt('podlogs', default=True, desc='collect logs from each pod'), + PluginOpt('podlogs-filter', default='', val_type=str, + desc='only collect logs from pods matching this pattern'), + PluginOpt('only-namespaces', default='', val_type=str, + desc='colon-delimited list of namespaces to collect from'), + PluginOpt('add-namespaces', default='', val_type=str, + desc=('colon-delimited list of namespaces to add to the ' + 'default collection list')) ] def _check_oc_function(self): diff -Nru sosreport-4.2/sos/report/plugins/openstack_ceilometer.py sosreport-4.3/sos/report/plugins/openstack_ceilometer.py --- sosreport-4.2/sos/report/plugins/openstack_ceilometer.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_ceilometer.py 2022-02-15 04:20:20.000000000 +0000 @@ -20,8 +20,6 @@ short_desc = 'Openstack Ceilometer' plugin_name = "openstack_ceilometer" profiles = ('openstack', 'openstack_controller', 'openstack_compute') - - option_list = [] var_puppet_gen = "/var/lib/config-data/puppet-generated/ceilometer" def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/openstack_database.py sosreport-4.3/sos/report/plugins/openstack_database.py --- sosreport-4.2/sos/report/plugins/openstack_database.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_database.py 2022-02-15 04:20:20.000000000 +0000 @@ -11,7 +11,7 @@ import re -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class OpenStackDatabase(Plugin): @@ -21,8 +21,8 @@ profiles = ('openstack', 'openstack_controller') option_list = [ - ('dump', 'Dump select databases to a SQL file', 'slow', False), - ('dumpall', 'Dump ALL databases to a SQL file', 'slow', False) + PluginOpt('dump', default=False, desc='Dump select databases'), + PluginOpt('dumpall', default=False, desc='Dump ALL databases') ] databases = [ @@ -37,36 +37,28 @@ ] def setup(self): - - in_container = False # determine if we're running databases on the host or in a container _db_containers = [ 'galera-bundle-.*', # overcloud 'mysql' # undercloud ] + cname = None for container in _db_containers: cname = self.get_container_by_name(container) - if cname is not None: - in_container = True + if cname: break - if in_container: - fname = "clustercheck_%s" % cname - cmd = self.fmt_container_cmd(cname, 'clustercheck') - self.add_cmd_output(cmd, timeout=15, suggest_filename=fname) - else: - self.add_cmd_output('clustercheck', timeout=15) + fname = "clustercheck_%s" % cname if cname else None + self.add_cmd_output('clustercheck', container=cname, timeout=15, + suggest_filename=fname) if self.get_option('dump') or self.get_option('dumpall'): db_dump = self.get_mysql_db_string(container=cname) db_cmd = "mysqldump --opt %s" % db_dump - if in_container: - db_cmd = self.fmt_container_cmd(cname, db_cmd) - self.add_cmd_output(db_cmd, suggest_filename='mysql_dump.sql', - sizelimit=0) + sizelimit=0, container=cname) def get_mysql_db_string(self, container=None): diff -Nru sosreport-4.2/sos/report/plugins/openstack_designate.py sosreport-4.3/sos/report/plugins/openstack_designate.py --- sosreport-4.2/sos/report/plugins/openstack_designate.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_designate.py 2022-02-15 04:20:20.000000000 +0000 @@ -20,12 +20,10 @@ def setup(self): # collect current pool config - pools_cmd = self.fmt_container_cmd( - self.get_container_by_name(".*designate_central"), - "designate-manage pool generate_file --file /dev/stdout") self.add_cmd_output( - pools_cmd, + "designate-manage pool generate_file --file /dev/stdout", + container=self.get_container_by_name(".*designate_central"), suggest_filename="openstack_designate_current_pools.yaml" ) diff -Nru sosreport-4.2/sos/report/plugins/openstack_glance.py sosreport-4.3/sos/report/plugins/openstack_glance.py --- sosreport-4.2/sos/report/plugins/openstack_glance.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_glance.py 2022-02-15 04:20:20.000000000 +0000 @@ -23,7 +23,6 @@ profiles = ('openstack', 'openstack_controller') containers = ('glance_api',) - option_list = [] var_puppet_gen = "/var/lib/config-data/puppet-generated/glance_api" service_name = "openstack-glance-api.service" diff -Nru sosreport-4.2/sos/report/plugins/openstack_heat.py sosreport-4.3/sos/report/plugins/openstack_heat.py --- sosreport-4.2/sos/report/plugins/openstack_heat.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_heat.py 2022-02-15 04:20:20.000000000 +0000 @@ -19,8 +19,6 @@ plugin_name = "openstack_heat" profiles = ('openstack', 'openstack_controller') containers = ('.*heat_api',) - - option_list = [] var_puppet_gen = "/var/lib/config-data/puppet-generated/heat" service_name = "openstack-heat-api.service" diff -Nru sosreport-4.2/sos/report/plugins/openstack_horizon.py sosreport-4.3/sos/report/plugins/openstack_horizon.py --- sosreport-4.2/sos/report/plugins/openstack_horizon.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_horizon.py 2022-02-15 04:20:20.000000000 +0000 @@ -20,7 +20,6 @@ plugin_name = "openstack_horizon" profiles = ('openstack', 'openstack_controller') - option_list = [] var_puppet_gen = "/var/lib/config-data/puppet-generated" def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/openstack_instack.py sosreport-4.3/sos/report/plugins/openstack_instack.py --- sosreport-4.2/sos/report/plugins/openstack_instack.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_instack.py 2022-02-15 04:20:20.000000000 +0000 @@ -68,7 +68,7 @@ p = uc_config.get(opt) if p: if not os.path.isabs(p): - p = os.path.join('/home/stack', p) + p = self.path_join('/home/stack', p) self.add_copy_spec(p) except Exception: pass diff -Nru sosreport-4.2/sos/report/plugins/openstack_ironic.py sosreport-4.3/sos/report/plugins/openstack_ironic.py --- sosreport-4.2/sos/report/plugins/openstack_ironic.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_ironic.py 2022-02-15 04:20:20.000000000 +0000 @@ -80,8 +80,7 @@ 'ironic_pxe_tftp', 'ironic_neutron_agent', 'ironic_conductor', 'ironic_api']: if self.container_exists('.*' + container_name): - self.add_cmd_output(self.fmt_container_cmd(container_name, - 'rpm -qa')) + self.add_cmd_output('rpm -qa', container=container_name) else: self.conf_list = [ diff -Nru sosreport-4.2/sos/report/plugins/openstack_keystone.py sosreport-4.3/sos/report/plugins/openstack_keystone.py --- sosreport-4.2/sos/report/plugins/openstack_keystone.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_keystone.py 2022-02-15 04:20:20.000000000 +0000 @@ -9,7 +9,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) import os @@ -19,7 +20,10 @@ plugin_name = "openstack_keystone" profiles = ('openstack', 'openstack_controller') - option_list = [("nopw", "dont gathers keystone passwords", "slow", True)] + option_list = [ + PluginOpt('nopw', default=True, + desc='do not collect keystone passwords') + ] var_puppet_gen = "/var/lib/config-data/puppet-generated/keystone" def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/openstack_manila.py sosreport-4.3/sos/report/plugins/openstack_manila.py --- sosreport-4.2/sos/report/plugins/openstack_manila.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_manila.py 2022-02-15 04:20:20.000000000 +0000 @@ -17,8 +17,6 @@ plugin_name = "openstack_manila" profiles = ('openstack', 'openstack_controller') containers = ('.*manila_api',) - option_list = [] - var_puppet_gen = "/var/lib/config-data/puppet-generated/manila" def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/openstack_nova.py sosreport-4.3/sos/report/plugins/openstack_nova.py --- sosreport-4.2/sos/report/plugins/openstack_nova.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_nova.py 2022-02-15 04:20:20.000000000 +0000 @@ -103,7 +103,7 @@ "nova-scheduler.log*" ] for novalog in novalogs: - self.add_copy_spec(os.path.join(novadir, novalog)) + self.add_copy_spec(self.path_join(novadir, novalog)) self.add_copy_spec([ "/etc/nova/", diff -Nru sosreport-4.2/sos/report/plugins/openstack_octavia.py sosreport-4.3/sos/report/plugins/openstack_octavia.py --- sosreport-4.2/sos/report/plugins/openstack_octavia.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_octavia.py 2022-02-15 04:20:20.000000000 +0000 @@ -5,7 +5,7 @@ # version 2 of the GNU General Public License. # # See the LICENSE file in the source distribution for further information. - +import os from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin @@ -62,35 +62,50 @@ ]) # commands - self.add_cmd_output('openstack loadbalancer list', - subdir='loadbalancer') + vars_all = [p in os.environ for p in [ + 'OS_USERNAME', 'OS_PASSWORD']] + + vars_any = [p in os.environ for p in [ + 'OS_TENANT_NAME', 'OS_PROJECT_NAME']] + + if not (all(vars_all) and any(vars_any)) and not \ + (self.is_installed("python2-octaviaclient") or + self.is_installed("python3-octaviaclient")): + self.soslog.warning("Not all environment variables set or " + "octavia client package not installed." + "Source the environment file for the " + "user intended to connect to the " + "OpenStack environment and install " + "octavia client package.") + else: + self.add_cmd_output('openstack loadbalancer list', + subdir='loadbalancer') - for res in self.resources: - # get a list for each resource type - self.add_cmd_output('openstack loadbalancer %s list' % res, - subdir=res) + for res in self.resources: + # get a list for each resource type + self.add_cmd_output('openstack loadbalancer %s list' % res, + subdir=res) + + # get details from each resource + cmd = "openstack loadbalancer %s list -f value -c id" % res + ret = self.exec_cmd(cmd) + if ret['status'] == 0: + for ent in ret['output'].splitlines(): + ent = ent.split()[0] + self.add_cmd_output( + "openstack loadbalancer %s show %s" % (res, ent), + subdir=res) - # get details from each resource - cmd = "openstack loadbalancer %s list -f value -c id" % res + # get capability details from each provider + cmd = "openstack loadbalancer provider list -f value -c name" ret = self.exec_cmd(cmd) if ret['status'] == 0: - for ent in ret['output'].splitlines(): - ent = ent.split()[0] + for p in ret['output'].splitlines(): + p = p.split()[0] self.add_cmd_output( - "openstack loadbalancer %s show %s" % (res, ent), - subdir=res - ) - - # get capability details from each provider - cmd = "openstack loadbalancer provider list -f value -c name" - ret = self.exec_cmd(cmd) - if ret['status'] == 0: - for p in ret['output'].splitlines(): - p = p.split()[0] - self.add_cmd_output( - "openstack loadbalancer provider capability list %s" % p, - subdir='provider_capability' - ) + "openstack loadbalancer provider capability list" + " %s" % p, + subdir='provider_capability') def postproc(self): protect_keys = [ diff -Nru sosreport-4.2/sos/report/plugins/openstack_sahara.py sosreport-4.3/sos/report/plugins/openstack_sahara.py --- sosreport-4.2/sos/report/plugins/openstack_sahara.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_sahara.py 2022-02-15 04:20:20.000000000 +0000 @@ -16,8 +16,6 @@ short_desc = 'OpenStack Sahara' plugin_name = 'openstack_sahara' profiles = ('openstack', 'openstack_controller') - - option_list = [] var_puppet_gen = "/var/lib/config-data/puppet-generated/sahara" def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/openstack_swift.py sosreport-4.3/sos/report/plugins/openstack_swift.py --- sosreport-4.2/sos/report/plugins/openstack_swift.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_swift.py 2022-02-15 04:20:20.000000000 +0000 @@ -20,8 +20,6 @@ plugin_name = "openstack_swift" profiles = ('openstack', 'openstack_controller') - option_list = [] - var_puppet_gen = "/var/lib/config-data/puppet-generated" def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/openstack_trove.py sosreport-4.3/sos/report/plugins/openstack_trove.py --- sosreport-4.2/sos/report/plugins/openstack_trove.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openstack_trove.py 2022-02-15 04:20:20.000000000 +0000 @@ -18,8 +18,6 @@ plugin_name = "openstack_trove" profiles = ('openstack', 'openstack_controller') - option_list = [] - var_puppet_gen = "/var/lib/config-data/puppet-generated/trove" def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/openvswitch.py sosreport-4.3/sos/report/plugins/openvswitch.py --- sosreport-4.2/sos/report/plugins/openvswitch.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/openvswitch.py 2022-02-15 04:20:20.000000000 +0000 @@ -10,7 +10,6 @@ from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin -from os.path import join as path_join from os import environ import re @@ -65,7 +64,9 @@ log_dirs.append(environ.get('OVS_LOGDIR')) if not all_logs: - self.add_copy_spec([path_join(ld, '*.log') for ld in log_dirs]) + self.add_copy_spec([ + self.path_join(ld, '*.log') for ld in log_dirs + ]) else: self.add_copy_spec(log_dirs) @@ -75,12 +76,19 @@ "/run/openvswitch/ovs-monitor-ipsec.pid" ]) + self.add_copy_spec([ + self.path_join('/usr/local/etc/openvswitch', 'conf.db'), + self.path_join('/etc/openvswitch', 'conf.db'), + self.path_join('/var/lib/openvswitch', 'conf.db'), + ]) + ovs_dbdir = environ.get('OVS_DBDIR') + if ovs_dbdir: + self.add_copy_spec(self.path_join(ovs_dbdir, 'conf.db')) + self.add_cmd_output([ # The '-t 5' adds an upper bound on how long to wait to connect # to the Open vSwitch server, avoiding hangs when running sos. "ovs-vsctl -t 5 show", - # Gather the database. - "ovsdb-client -f list dump", # List the contents of important runtime directories "ls -laZ /run/openvswitch", "ls -laZ /dev/hugepages/", @@ -131,7 +139,13 @@ # Capture OVS offload enabled flows "ovs-dpctl dump-flows --name -m type=offloaded", # Capture OVS slowdatapth flows - "ovs-dpctl dump-flows --name -m type=ovs" + "ovs-dpctl dump-flows --name -m type=ovs", + # Capture dpcls implementations + "ovs-appctl dpif-netdev/subtable-lookup-prio-get", + # Capture dpif implementations + "ovs-appctl dpif-netdev/dpif-impl-get", + # Capture miniflow extract implementations + "ovs-appctl dpif-netdev/miniflow-parser-get" ]) # Gather systemd services logs @@ -200,6 +214,7 @@ # Gather additional output for each OVS bridge on the host. br_list_result = self.collect_cmd_output("ovs-vsctl -t 5 list-br") + ofp_ver_result = self.collect_cmd_output("ovs-ofctl -t 5 --version") if br_list_result['status'] == 0: for br in br_list_result['output'].splitlines(): self.add_cmd_output([ @@ -226,6 +241,16 @@ "OpenFlow15" ] + # Flow protocol hex identifiers + ofp_versions = { + 0x01: "OpenFlow10", + 0x02: "OpenFlow11", + 0x03: "OpenFlow12", + 0x04: "OpenFlow13", + 0x05: "OpenFlow14", + 0x06: "OpenFlow15", + } + # List protocols currently in use, if any ovs_list_bridge_cmd = "ovs-vsctl -t 5 list bridge %s" % br br_info = self.collect_cmd_output(ovs_list_bridge_cmd) @@ -236,6 +261,21 @@ br_protos_ln = line[line.find("[")+1:line.find("]")] br_protos = br_protos_ln.replace('"', '').split(", ") + # If 'list bridge' yeilded no protocols, use the range of + # protocols enabled by default on this version of ovs. + if br_protos == [''] and ofp_ver_result['output']: + ofp_version_range = ofp_ver_result['output'].splitlines() + ver_range = [] + + for line in ofp_version_range: + if "OpenFlow versions" in line: + v = line.split("OpenFlow versions ")[1].split(":") + ver_range = range(int(v[0], 16), int(v[1], 16)+1) + + for protocol in ver_range: + if protocol in ofp_versions: + br_protos.append(ofp_versions[protocol]) + # Collect flow information for relevant protocol versions only for flow in flow_versions: if flow in br_protos: @@ -244,6 +284,7 @@ "ovs-ofctl -O %s dump-groups %s" % (flow, br), "ovs-ofctl -O %s dump-group-stats %s" % (flow, br), "ovs-ofctl -O %s dump-flows %s" % (flow, br), + "ovs-ofctl -O %s dump-tlv-map %s" % (flow, br), "ovs-ofctl -O %s dump-ports-desc %s" % (flow, br) ]) diff -Nru sosreport-4.2/sos/report/plugins/origin.py sosreport-4.3/sos/report/plugins/origin.py --- sosreport-4.2/sos/report/plugins/origin.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/origin.py 2022-02-15 04:20:20.000000000 +0000 @@ -40,15 +40,6 @@ files = None # file lists assigned after path setup below profiles = ('openshift',) - option_list = [ - ("diag", "run 'oc adm diagnostics' to collect its output", - 'fast', True), - ("diag-prevent", "set --prevent-modification on 'oc adm diagnostics'", - 'fast', True), - ("all-namespaces", "collect dc output for all namespaces", "fast", - False) - ] - master_base_dir = "/etc/origin/master" node_base_dir = "/etc/origin/node" master_cfg = os.path.join(master_base_dir, "master-config.yaml") @@ -78,20 +69,21 @@ def is_static_etcd(self): """Determine if we are on a node running etcd""" - return self.path_exists(os.path.join(self.static_pod_dir, "etcd.yaml")) + return self.path_exists(self.path_join(self.static_pod_dir, + "etcd.yaml")) def is_static_pod_compatible(self): """Determine if a node is running static pods""" return self.path_exists(self.static_pod_dir) def setup(self): - bstrap_node_cfg = os.path.join(self.node_base_dir, - "bootstrap-" + self.node_cfg_file) - bstrap_kubeconfig = os.path.join(self.node_base_dir, - "bootstrap.kubeconfig") - node_certs = os.path.join(self.node_base_dir, "certs", "*") - node_client_ca = os.path.join(self.node_base_dir, "client-ca.crt") - admin_cfg = os.path.join(self.master_base_dir, "admin.kubeconfig") + bstrap_node_cfg = self.path_join(self.node_base_dir, + "bootstrap-" + self.node_cfg_file) + bstrap_kubeconfig = self.path_join(self.node_base_dir, + "bootstrap.kubeconfig") + node_certs = self.path_join(self.node_base_dir, "certs", "*") + node_client_ca = self.path_join(self.node_base_dir, "client-ca.crt") + admin_cfg = self.path_join(self.master_base_dir, "admin.kubeconfig") oc_cmd_admin = "%s --config=%s" % ("oc", admin_cfg) static_pod_logs_cmd = "master-logs" @@ -101,16 +93,20 @@ self.add_copy_spec([ self.master_cfg, self.master_env, - os.path.join(self.master_base_dir, "*.crt"), + self.path_join(self.master_base_dir, "*.crt"), ]) if self.is_static_pod_compatible(): - self.add_copy_spec(os.path.join(self.static_pod_dir, "*.yaml")) + self.add_copy_spec(self.path_join(self.static_pod_dir, + "*.yaml")) self.add_cmd_output([ "%s api api" % static_pod_logs_cmd, "%s controllers controllers" % static_pod_logs_cmd, ]) + if self.is_static_etcd(): + self.add_cmd_output("%s etcd etcd" % static_pod_logs_cmd) + # TODO: some thoughts about information that might also be useful # to collect. However, these are maybe not needed in general # and/or present some challenges (scale, sensitive, ...) and need @@ -129,9 +125,9 @@ # is already collected by the Kubernetes plugin subcmds = [ - "describe projects", "adm top images", - "adm top imagestreams" + "adm top imagestreams", + "adm top nodes" ] self.add_cmd_output([ @@ -148,29 +144,23 @@ '%s get -o json %s' % (oc_cmd_admin, jcmd) for jcmd in jcmds ]) - if self.get_option('all-namespaces'): - ocn = self.exec_cmd('%s get namespaces' % oc_cmd_admin) - ns_output = ocn['output'].splitlines()[1:] - nmsps = [n.split()[0] for n in ns_output if n] - else: - nmsps = [ - 'default', - 'openshift-web-console', - 'openshift-ansible-service-broker' - ] + nmsps = [ + 'default', + 'openshift-web-console', + 'openshift-ansible-service-broker', + 'openshift-sdn', + 'openshift-console' + ] self.add_cmd_output([ - '%s get -o json dc -n %s' % (oc_cmd_admin, n) for n in nmsps + '%s get -o json deploymentconfig,deployment,daemonsets -n %s' + % (oc_cmd_admin, n) for n in nmsps ]) - if self.get_option('diag'): - diag_cmd = "%s adm diagnostics -l 0" % oc_cmd_admin - if self.get_option('diag-prevent'): - diag_cmd += " --prevent-modification=true" - self.add_cmd_output(diag_cmd) - self.add_journal(units=["atomic-openshift-master", - "atomic-openshift-master-api", - "atomic-openshift-master-controllers"]) + if not self.is_static_pod_compatible(): + self.add_journal(units=["atomic-openshift-master", + "atomic-openshift-master-api", + "atomic-openshift-master-controllers"]) # get logs from the infrastruture pods running in the default ns pods = self.exec_cmd("%s get pod -o name -n default" @@ -189,16 +179,13 @@ node_client_ca, bstrap_node_cfg, bstrap_kubeconfig, - os.path.join(self.node_base_dir, "*.crt"), - os.path.join(self.node_base_dir, "resolv.conf"), - os.path.join(self.node_base_dir, "node-dnsmasq.conf"), + self.path_join(self.node_base_dir, "*.crt"), + self.path_join(self.node_base_dir, "resolv.conf"), + self.path_join(self.node_base_dir, "node-dnsmasq.conf"), ]) self.add_journal(units="atomic-openshift-node") - if self.is_static_etcd(): - self.add_cmd_output("%s etcd etcd" % static_pod_logs_cmd) - def postproc(self): # Clear env values from objects that can contain sensitive data # Sample JSON content: diff -Nru sosreport-4.2/sos/report/plugins/ovirt_engine_backup.py sosreport-4.3/sos/report/plugins/ovirt_engine_backup.py --- sosreport-4.2/sos/report/plugins/ovirt_engine_backup.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ovirt_engine_backup.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,8 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -import os -from sos.report.plugins import (Plugin, RedHatPlugin) +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt from datetime import datetime @@ -20,20 +19,20 @@ packages = ("ovirt-engine-tools-backup",) plugin_name = "ovirt_engine_backup" option_list = [ - ("backupdir", "Directory where the backup is generated", - "fast", "/var/lib/ovirt-engine-backup"), - ("tmpdir", "Directory where the intermediate files are generated", - "fast", '/tmp'), + PluginOpt('backupdir', default='/var/lib/ovirt-engine-backup', + desc='Directory where backups are generated'), + PluginOpt('tmpdir', default='/tmp', + desc='temp dir to use for engine-backup') ] profiles = ("virt",) def setup(self): now = datetime.now().strftime("%Y%m%d%H%M%S") - backup_filename = os.path.join( + backup_filename = self.path_join( self.get_option("backupdir"), "engine-db-backup-%s.tar.gz" % (now) ) - log_filename = os.path.join( + log_filename = self.path_join( self.get_option("backupdir"), "engine-db-backup-%s.log" % (now) ) diff -Nru sosreport-4.2/sos/report/plugins/ovirt.py sosreport-4.3/sos/report/plugins/ovirt.py --- sosreport-4.2/sos/report/plugins/ovirt.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ovirt.py 2022-02-15 04:20:20.000000000 +0000 @@ -16,7 +16,7 @@ import signal -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt from sos.utilities import is_executable @@ -55,12 +55,12 @@ ) option_list = [ - ('jbosstrace', 'Enable oVirt Engine JBoss stack trace collection', - '', True), - ('sensitive_keys', 'Sensitive keys to be masked', - '', DEFAULT_SENSITIVE_KEYS), - ('heapdump', 'Collect heap dumps from /var/log/ovirt-engine/dump/', - '', False) + PluginOpt('jbosstrace', default=True, + desc='Enable oVirt Engine JBoss stack trace collection'), + PluginOpt('sensitive_keys', default=DEFAULT_SENSITIVE_KEYS, + desc='Sensitive keys to be masked in post-processing'), + PluginOpt('heapdump', default=False, + desc='Collect heap dumps from /var/log/ovirt-engine/dump/') ] def setup(self): @@ -216,7 +216,7 @@ "isouploader.conf" ] for conf_file in passwd_files: - conf_path = os.path.join("/etc/ovirt-engine", conf_file) + conf_path = self.path_join("/etc/ovirt-engine", conf_file) self.do_file_sub( conf_path, r"passwd=(.*)", diff -Nru sosreport-4.2/sos/report/plugins/ovn_central.py sosreport-4.3/sos/report/plugins/ovn_central.py --- sosreport-4.2/sos/report/plugins/ovn_central.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ovn_central.py 2022-02-15 04:20:20.000000000 +0000 @@ -42,7 +42,7 @@ return else: try: - with open(filename, 'r') as f: + with open(self.path_join(filename), 'r') as f: try: db = json.load(f) except Exception: @@ -71,19 +71,27 @@ ovs_rundir = os.environ.get('OVS_RUNDIR') for pidfile in ['ovnnb_db.pid', 'ovnsb_db.pid', 'ovn-northd.pid']: self.add_copy_spec([ - os.path.join('/var/lib/openvswitch/ovn', pidfile), - os.path.join('/usr/local/var/run/openvswitch', pidfile), - os.path.join('/run/openvswitch/', pidfile), + self.path_join('/var/lib/openvswitch/ovn', pidfile), + self.path_join('/usr/local/var/run/openvswitch', pidfile), + self.path_join('/run/openvswitch/', pidfile), ]) if ovs_rundir: - self.add_copy_spec(os.path.join(ovs_rundir, pidfile)) + self.add_copy_spec(self.path_join(ovs_rundir, pidfile)) if self.get_option("all_logs"): self.add_copy_spec("/var/log/ovn/") else: self.add_copy_spec("/var/log/ovn/*.log") + # ovsdb nb/sb cluster status commands + ovsdb_cmds = [ + 'ovs-appctl -t {} cluster/status OVN_Northbound'.format( + self.ovn_nbdb_sock_path), + 'ovs-appctl -t {} cluster/status OVN_Southbound'.format( + self.ovn_sbdb_sock_path), + ] + # Some user-friendly versions of DB output nbctl_cmds = [ 'ovn-nbctl show', @@ -104,54 +112,57 @@ schema_dir = '/usr/share/openvswitch' - nb_tables = self.get_tables_from_schema(os.path.join( + nb_tables = self.get_tables_from_schema(self.path_join( schema_dir, 'ovn-nb.ovsschema')) self.add_database_output(nb_tables, nbctl_cmds, 'ovn-nbctl') - cmds = nbctl_cmds + cmds = ovsdb_cmds + cmds += nbctl_cmds # Can only run sbdb commands if we are the leader co = {'cmd': "ovs-appctl -t {} cluster/status OVN_Southbound". format(self.ovn_sbdb_sock_path), "output": "Leader: self"} if self.test_predicate(self, pred=SoSPredicate(self, cmd_outputs=co)): - sb_tables = self.get_tables_from_schema(os.path.join( + sb_tables = self.get_tables_from_schema(self.path_join( schema_dir, 'ovn-sb.ovsschema'), ['Logical_Flow']) self.add_database_output(sb_tables, sbctl_cmds, 'ovn-sbctl') cmds += sbctl_cmds # If OVN is containerized, we need to run the above commands inside # the container. - cmds = [ - self.fmt_container_cmd(self._container_name, cmd) for cmd in cmds - ] - self.add_cmd_output(cmds, foreground=True) + self.add_cmd_output( + cmds, foreground=True, container=self._container_name + ) self.add_copy_spec("/etc/sysconfig/ovn-northd") ovs_dbdir = os.environ.get('OVS_DBDIR') for dbfile in ['ovnnb_db.db', 'ovnsb_db.db']: self.add_copy_spec([ - os.path.join('/var/lib/openvswitch/ovn', dbfile), - os.path.join('/usr/local/etc/openvswitch', dbfile), - os.path.join('/etc/openvswitch', dbfile), - os.path.join('/var/lib/openvswitch', dbfile), + self.path_join('/var/lib/openvswitch/ovn', dbfile), + self.path_join('/usr/local/etc/openvswitch', dbfile), + self.path_join('/etc/openvswitch', dbfile), + self.path_join('/var/lib/openvswitch', dbfile), + self.path_join('/var/lib/ovn/etc', dbfile) ]) if ovs_dbdir: - self.add_copy_spec(os.path.join(ovs_dbdir, dbfile)) + self.add_copy_spec(self.path_join(ovs_dbdir, dbfile)) self.add_journal(units="ovn-northd") class RedHatOVNCentral(OVNCentral, RedHatPlugin): - packages = ('openvswitch-ovn-central', 'ovn2.*-central', ) + packages = ('openvswitch-ovn-central', 'ovn.*-central', ) + ovn_nbdb_sock_path = '/var/run/openvswitch/ovnnb_db.ctl' ovn_sbdb_sock_path = '/var/run/openvswitch/ovnsb_db.ctl' class DebianOVNCentral(OVNCentral, DebianPlugin, UbuntuPlugin): packages = ('ovn-central', ) + ovn_nbdb_sock_path = '/var/run/ovn/ovnnb_db.ctl' ovn_sbdb_sock_path = '/var/run/ovn/ovnsb_db.ctl' diff -Nru sosreport-4.2/sos/report/plugins/ovn_host.py sosreport-4.3/sos/report/plugins/ovn_host.py --- sosreport-4.2/sos/report/plugins/ovn_host.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ovn_host.py 2022-02-15 04:20:20.000000000 +0000 @@ -35,7 +35,7 @@ else: self.add_copy_spec("/var/log/ovn/*.log") - self.add_copy_spec([os.path.join(pp, pidfile) for pp in pid_paths]) + self.add_copy_spec([self.path_join(pp, pidfile) for pp in pid_paths]) self.add_copy_spec('/etc/sysconfig/ovn-controller') @@ -49,13 +49,13 @@ def check_enabled(self): return (any([self.path_isfile( - os.path.join(pp, pidfile)) for pp in pid_paths]) or + self.path_join(pp, pidfile)) for pp in pid_paths]) or super(OVNHost, self).check_enabled()) class RedHatOVNHost(OVNHost, RedHatPlugin): - packages = ('openvswitch-ovn-host', 'ovn2.*-host', ) + packages = ('openvswitch-ovn-host', 'ovn.*-host', ) class DebianOVNHost(OVNHost, DebianPlugin, UbuntuPlugin): diff -Nru sosreport-4.2/sos/report/plugins/pacemaker.py sosreport-4.3/sos/report/plugins/pacemaker.py --- sosreport-4.2/sos/report/plugins/pacemaker.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/pacemaker.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) from datetime import datetime, timedelta import re @@ -23,8 +24,10 @@ ) option_list = [ - ("crm_from", "specify the start time for crm_report", "fast", ''), - ("crm_scrub", "enable password scrubbing for crm_report", "", True), + PluginOpt('crm_from', default='', val_type=str, + desc='specfiy the start time for crm_report'), + PluginOpt('crm_scrub', default=True, + desc='enable crm_report password scrubbing') ] envfile = "" @@ -126,7 +129,7 @@ class DebianPacemaker(Pacemaker, DebianPlugin, UbuntuPlugin): def setup(self): - self.envfile = "/etc/default/pacemaker" + self.envfile = self.path_join("/etc/default/pacemaker") self.setup_crm_shell() self.setup_pcs() super(DebianPacemaker, self).setup() @@ -138,7 +141,7 @@ class RedHatPacemaker(Pacemaker, RedHatPlugin): def setup(self): - self.envfile = "/etc/sysconfig/pacemaker" + self.envfile = self.path_join("/etc/sysconfig/pacemaker") self.setup_pcs() self.add_copy_spec("/etc/sysconfig/sbd") super(RedHatPacemaker, self).setup() diff -Nru sosreport-4.2/sos/report/plugins/pcp.py sosreport-4.3/sos/report/plugins/pcp.py --- sosreport-4.2/sos/report/plugins/pcp.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/pcp.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin +from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, PluginOpt import os from socket import gethostname @@ -25,8 +25,10 @@ # size-limit of PCP logger and manager data collected by default (MB) option_list = [ - ("pmmgrlogs", "size-limit in MB of pmmgr logs", "", 100), - ("pmloggerfiles", "number of newest pmlogger files to grab", "", 12), + PluginOpt('pmmgrlogs', default=100, + desc='size limit in MB of pmmgr logs'), + PluginOpt('pmloggerfiles', default=12, + desc='number of pmlogger files to collect') ] pcp_sysconf_dir = None @@ -39,7 +41,7 @@ total_size = 0 for dirpath, dirnames, filenames in os.walk(path): for f in filenames: - fp = os.path.join(dirpath, f) + fp = self.path_join(dirpath, f) total_size += os.path.getsize(fp) return total_size @@ -84,7 +86,7 @@ # unconditionally. Obviously if someone messes up their /etc/pcp.conf # in a ridiculous way (i.e. setting PCP_SYSCONF_DIR to '/') this will # break badly. - var_conf_dir = os.path.join(self.pcp_var_dir, 'config') + var_conf_dir = self.path_join(self.pcp_var_dir, 'config') self.add_copy_spec([ self.pcp_sysconf_dir, self.pcp_conffile, @@ -96,10 +98,10 @@ # rpms. It does not make up for a lot of size but it contains many # files self.add_forbidden_path([ - os.path.join(var_conf_dir, 'pmchart'), - os.path.join(var_conf_dir, 'pmlogconf'), - os.path.join(var_conf_dir, 'pmieconf'), - os.path.join(var_conf_dir, 'pmlogrewrite') + self.path_join(var_conf_dir, 'pmchart'), + self.path_join(var_conf_dir, 'pmlogconf'), + self.path_join(var_conf_dir, 'pmieconf'), + self.path_join(var_conf_dir, 'pmlogrewrite') ]) # Take PCP_LOG_DIR/pmlogger/`hostname` + PCP_LOG_DIR/pmmgr/`hostname` @@ -119,13 +121,13 @@ # we would collect everything if self.pcp_hostname != '': # collect pmmgr logs up to 'pmmgrlogs' size limit - path = os.path.join(self.pcp_log_dir, 'pmmgr', - self.pcp_hostname, '*') + path = self.path_join(self.pcp_log_dir, 'pmmgr', + self.pcp_hostname, '*') self.add_copy_spec(path, sizelimit=self.sizelimit, tailit=False) # collect newest pmlogger logs up to 'pmloggerfiles' count files_collected = 0 - path = os.path.join(self.pcp_log_dir, 'pmlogger', - self.pcp_hostname, '*') + path = self.path_join(self.pcp_log_dir, 'pmlogger', + self.pcp_hostname, '*') pmlogger_ls = self.exec_cmd("ls -t1 %s" % path) if pmlogger_ls['status'] == 0: for line in pmlogger_ls['output'].splitlines(): @@ -136,15 +138,15 @@ self.add_copy_spec([ # Collect PCP_LOG_DIR/pmcd and PCP_LOG_DIR/NOTICES - os.path.join(self.pcp_log_dir, 'pmcd'), - os.path.join(self.pcp_log_dir, 'NOTICES*'), + self.path_join(self.pcp_log_dir, 'pmcd'), + self.path_join(self.pcp_log_dir, 'NOTICES*'), # Collect PCP_VAR_DIR/pmns - os.path.join(self.pcp_var_dir, 'pmns'), + self.path_join(self.pcp_var_dir, 'pmns'), # Also collect any other log and config files # (as suggested by fche) - os.path.join(self.pcp_log_dir, '*/*.log*'), - os.path.join(self.pcp_log_dir, '*/*/*.log*'), - os.path.join(self.pcp_log_dir, '*/*/config*') + self.path_join(self.pcp_log_dir, '*/*.log*'), + self.path_join(self.pcp_log_dir, '*/*/*.log*'), + self.path_join(self.pcp_log_dir, '*/*/config*') ]) # Collect a summary for the current day diff -Nru sosreport-4.2/sos/report/plugins/perccli.py sosreport-4.3/sos/report/plugins/perccli.py --- sosreport-4.2/sos/report/plugins/perccli.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/perccli.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,58 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt + + +class PercCLI(Plugin, IndependentPlugin): + + short_desc = 'PowerEdge RAID Controller management' + + plugin_name = 'perccli' + profiles = ('system', 'storage', 'hardware',) + packages = ('perccli',) + + option_list = [ + PluginOpt('json', default=False, desc='collect data in JSON format') + ] + + def setup(self): + cmd = '/opt/MegaRAID/perccli/perccli64' + subcmds = [ + 'show ctrlcount', + '/call show AliLog', + '/call show all', + '/call show termlog', + '/call/bbu show all', + '/call/cv show all', + '/call/dall show', + '/call/eall show all', + '/call/eall/sall show all', + '/call/sall show all', + '/call/vall show all', + ] + + json = ' J' if self.get_option('json') else '' + + logpath = self.get_cmd_output_path() + + for subcmd in subcmds: + self.add_cmd_output( + "%s %s%s" % (cmd, subcmd, json), + suggest_filename="perccli64_%s%s" % (subcmd, json), + runat=logpath) + + # /call show events need 'file=' option to get adapter info like below + # "Adapter: # - Number of Events: xxx". + subcmd = '/call show events' + self.add_cmd_output( + "%s %s file=/dev/stdout%s" % (cmd, subcmd, json), + suggest_filename="perccli64_%s%s" % (subcmd, json), + runat=logpath) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/podman.py sosreport-4.3/sos/report/plugins/podman.py --- sosreport-4.2/sos/report/plugins/podman.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/podman.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,10 +8,20 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin +from sos.report.plugins import Plugin, RedHatPlugin, UbuntuPlugin, PluginOpt class Podman(Plugin, RedHatPlugin, UbuntuPlugin): + """Podman is a daemonless container management engine, and this plugin is + meant to provide diagnostic information for both the engine and the + containers that podman is managing. + + General status information will be collected from podman commands, while + detailed inspections of certain components will provide more insight + into specific container problems. This detailed inspection is provided for + containers, images, networks, and volumes. Per-entity inspections will be + recorded in subdirs within sos_commands/podman/ for each of those types. + """ short_desc = 'Podman containers' plugin_name = 'podman' @@ -19,11 +29,22 @@ packages = ('podman',) option_list = [ - ("all", "enable capture for all containers, even containers " - "that have terminated", 'fast', False), - ("logs", "capture logs for running containers", - 'fast', False), - ("size", "capture image sizes for podman ps", 'slow', False) + PluginOpt('all', default=False, + desc='collect for all containers, even terminated ones', + long_desc=( + 'Enable collection for all containers that exist on the ' + 'system regardless of their running state. This may cause ' + 'a significant increase in sos archive size, especially ' + 'when combined with the \'logs\' option.')), + PluginOpt('logs', default=False, + desc='collect stdout/stderr logs for containers', + long_desc=( + 'Capture \'podman logs\' output for discovered containers.' + ' This may be useful or not depending on how/if the ' + 'container produces stdout/stderr output. Use cautiously ' + 'when also using the \'all\' option.')), + PluginOpt('size', default=False, + desc='collect image sizes for podman ps') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/postfix.py sosreport-4.3/sos/report/plugins/postfix.py --- sosreport-4.2/sos/report/plugins/postfix.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/postfix.py 2022-02-15 04:20:20.000000000 +0000 @@ -41,7 +41,7 @@ ] fp = [] try: - with open('/etc/postfix/main.cf', 'r') as cffile: + with open(self.path_join('/etc/postfix/main.cf'), 'r') as cffile: for line in cffile.readlines(): # ignore comments and take the first word after '=' if line.startswith('#'): diff -Nru sosreport-4.2/sos/report/plugins/postgresql.py sosreport-4.3/sos/report/plugins/postgresql.py --- sosreport-4.2/sos/report/plugins/postgresql.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/postgresql.py 2022-02-15 04:20:20.000000000 +0000 @@ -14,7 +14,8 @@ import os -from sos.report.plugins import (Plugin, UbuntuPlugin, DebianPlugin, SCLPlugin) +from sos.report.plugins import (Plugin, UbuntuPlugin, DebianPlugin, SCLPlugin, + PluginOpt) from sos.utilities import find @@ -30,12 +31,18 @@ password_warn_text = " (password visible in process listings)" option_list = [ - ('pghome', 'PostgreSQL server home directory.', '', '/var/lib/pgsql'), - ('username', 'username for pg_dump', '', 'postgres'), - ('password', 'password for pg_dump' + password_warn_text, '', ''), - ('dbname', 'database name to dump for pg_dump', '', ''), - ('dbhost', 'database hostname/IP (do not use unix socket)', '', ''), - ('dbport', 'database server port number', '', '5432') + PluginOpt('pghome', default='/var/lib/pgsql', + desc='psql server home directory'), + PluginOpt('username', default='postgres', val_type=str, + desc='username for pg_dump'), + PluginOpt('password', default='', val_type=str, + desc='password for pg_dump' + password_warn_text), + PluginOpt('dbname', default='', val_type=str, + desc='database name to dump with pg_dump'), + PluginOpt('dbhost', default='', val_type=str, + desc='database hostname/IP address (no unix sockets)'), + PluginOpt('dbport', default=5432, val_type=[int, str], + desc='database server listening port') ] def do_pg_dump(self, scl=None, filename="pgdump.tar"): @@ -117,7 +124,7 @@ # copy PG_VERSION and postmaster.opts for f in ["PG_VERSION", "postmaster.opts"]: - self.add_copy_spec(os.path.join(_dir, "data", f)) + self.add_copy_spec(self.path_join(_dir, "data", f)) class DebianPostgreSQL(PostgreSQL, DebianPlugin, UbuntuPlugin): diff -Nru sosreport-4.2/sos/report/plugins/powerpc.py sosreport-4.3/sos/report/plugins/powerpc.py --- sosreport-4.2/sos/report/plugins/powerpc.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/powerpc.py 2022-02-15 04:20:20.000000000 +0000 @@ -22,7 +22,7 @@ def setup(self): try: - with open('/proc/cpuinfo', 'r') as fp: + with open(self.path_join('/proc/cpuinfo'), 'r') as fp: contents = fp.read() ispSeries = "pSeries" in contents isPowerNV = "PowerNV" in contents diff -Nru sosreport-4.2/sos/report/plugins/processor.py sosreport-4.3/sos/report/plugins/processor.py --- sosreport-4.2/sos/report/plugins/processor.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/processor.py 2022-02-15 04:20:20.000000000 +0000 @@ -34,7 +34,13 @@ self.add_copy_spec([ "/proc/cpuinfo", "/sys/class/cpuid", - "/sys/devices/system/cpu" + ]) + # copy /sys/devices/system/cpu/cpuX with separately applied sizelimit + # this is required for systems with tens/hundreds of CPUs where the + # cumulative directory size exceeds 25MB or even 100MB. + cdirs = self.listdir('/sys/devices/system/cpu') + self.add_copy_spec([ + self.path_join('/sys/devices/system/cpu', cdir) for cdir in cdirs ]) self.add_cmd_output([ diff -Nru sosreport-4.2/sos/report/plugins/process.py sosreport-4.3/sos/report/plugins/process.py --- sosreport-4.2/sos/report/plugins/process.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/process.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ import re -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Process(Plugin, IndependentPlugin): @@ -19,13 +19,14 @@ profiles = ('system',) option_list = [ - ("lsof", "gathers information on all open files", "slow", True), - ("lsof-threads", "gathers threads' open file info if supported", - "slow", False), - ("smaps", "gathers all /proc/*/smaps files", "", False), - ("samples", "specify the number of samples that iotop will capture, " - "with an interval of 0.5 seconds between samples", "", "20"), - ("numprocs", "number of processes to collect /proc data of", '', 2048) + PluginOpt('lsof', default=True, desc='collect info on all open files'), + PluginOpt('lsof-threads', default=False, + desc='collect threads\' open file info if supported'), + PluginOpt('smaps', default=False, desc='collect /proc/*/smaps files'), + PluginOpt('samples', default=20, val_type=int, + desc='number of iotop samples to collect'), + PluginOpt('numprocs', default=2048, val_type=int, + desc='number of process to collect /proc data of') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/psacct.py sosreport-4.3/sos/report/plugins/psacct.py --- sosreport-4.2/sos/report/plugins/psacct.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/psacct.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) class Psacct(Plugin): @@ -15,8 +16,9 @@ plugin_name = "psacct" profiles = ('system',) - option_list = [("all", "collect all process accounting files", - "slow", False)] + option_list = [ + PluginOpt('all', default=False, desc='collect all accounting files') + ] packages = ("psacct", ) diff -Nru sosreport-4.2/sos/report/plugins/pulpcore.py sosreport-4.3/sos/report/plugins/pulpcore.py --- sosreport-4.2/sos/report/plugins/pulpcore.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/pulpcore.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt from pipes import quote from re import match @@ -21,7 +21,7 @@ commands = ("pulpcore-manager",) files = ("/etc/pulp/settings.py",) option_list = [ - ('task-days', 'days of tasks history', 'fast', 7) + PluginOpt('task-days', default=7, desc='days of task history') ] def parse_settings_config(self): diff -Nru sosreport-4.2/sos/report/plugins/pulp.py sosreport-4.3/sos/report/plugins/pulp.py --- sosreport-4.2/sos/report/plugins/pulp.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/pulp.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt from pipes import quote from re import match @@ -21,7 +21,8 @@ packages = ("pulp-server", "pulp-katello", "python3-pulpcore") files = ("/etc/pulp/settings.py",) option_list = [ - ('tasks', 'number of tasks to collect from DB queries', 'fast', 200) + PluginOpt('tasks', default=200, + desc='number of tasks to collect from DB queries') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/pxe.py sosreport-4.3/sos/report/plugins/pxe.py --- sosreport-4.2/sos/report/plugins/pxe.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/pxe.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) class Pxe(Plugin): @@ -14,8 +15,10 @@ short_desc = 'PXE service' plugin_name = "pxe" profiles = ('sysmgmt', 'network') - option_list = [("tftpboot", 'gathers content from the tftpboot path', - 'slow', False)] + option_list = [ + PluginOpt('tftpboot', default=False, + desc='collect content from tftpboot path') + ] class RedHatPxe(Pxe, RedHatPlugin): diff -Nru sosreport-4.2/sos/report/plugins/python.py sosreport-4.3/sos/report/plugins/python.py --- sosreport-4.2/sos/report/plugins/python.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/python.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) from sos.policies.distros.redhat import RHELPolicy import os import json @@ -43,8 +44,9 @@ packages = ('python', 'python36', 'python2', 'python3', 'platform-python') option_list = [ - ('hashes', "gather hashes for all python files", 'slow', - False)] + PluginOpt('hashes', default=False, + desc='collect hashes for all python files') + ] def setup(self): self.add_cmd_output(['python2 -V', 'python3 -V']) @@ -66,9 +68,9 @@ ] for py_path in py_paths: - for root, _, files in os.walk(py_path): + for root, _, files in os.walk(self.path_join(py_path)): for file_ in files: - filepath = os.path.join(root, file_) + filepath = self.path_join(root, file_) if filepath.endswith('.py'): try: with open(filepath, 'rb') as f: @@ -93,6 +95,7 @@ filepath ) - self.add_string_as_file(json.dumps(digests), 'digests.json') + self.add_string_as_file(json.dumps(digests), 'digests.json', + plug_dir=True) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/qpid_dispatch.py sosreport-4.3/sos/report/plugins/qpid_dispatch.py --- sosreport-4.2/sos/report/plugins/qpid_dispatch.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/qpid_dispatch.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt from socket import gethostname @@ -20,12 +20,16 @@ profiles = ('services',) packages = ('qdrouterd', 'qpid-dispatch-tools', 'qpid-dispatch-router') - option_list = [("port", "listening port to connect to", '', ""), - ("ssl-certificate", - "Path to file containing client SSL certificate", '', ""), - ("ssl-key", - "Path to file containing client SSL private key", '', ""), - ("ssl-trustfile", "trusted CA database file", '', "")] + option_list = [ + PluginOpt('port', default='', val_type=int, + desc='listening port to connect to'), + PluginOpt('ssl-certificate', default='', val_type=str, + desc='Path to file containing client SSL certificate'), + PluginOpt('ssl-key', default='', val_type=str, + desc='Path to file containing client SSL private key'), + PluginOpt('ssl-trustfile', default='', val_type=str, + desc='trusted CA database file') + ] def setup(self): """ performs data collection for qpid dispatch router """ diff -Nru sosreport-4.2/sos/report/plugins/qpid.py sosreport-4.3/sos/report/plugins/qpid.py --- sosreport-4.2/sos/report/plugins/qpid.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/qpid.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Qpid(Plugin, RedHatPlugin): @@ -17,12 +17,15 @@ profiles = ('services',) packages = ('qpidd', 'qpid-cpp-server', 'qpid-tools') - option_list = [("port", "listening port to connect to", '', ""), - ("ssl-certificate", - "Path to file containing client SSL certificate", '', ""), - ("ssl-key", - "Path to file containing client SSL private key", '', ""), - ("ssl", "enforce SSL / amqps connection", '', False)] + option_list = [ + PluginOpt('port', default='', val_type=int, + desc='listening port to connect to'), + PluginOpt('ssl-certificate', default='', val_type=str, + desc='Path to file containing client SSL certificate'), + PluginOpt('ssl-key', default='', val_type=str, + desc='Path to file containing client SSL private key'), + PluginOpt('ssl', default=False, desc='enforce SSL amqps connection') + ] def setup(self): """ performs data collection for qpid broker """ diff -Nru sosreport-4.2/sos/report/plugins/rabbitmq.py sosreport-4.3/sos/report/plugins/rabbitmq.py --- sosreport-4.2/sos/report/plugins/rabbitmq.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/rabbitmq.py 2022-02-15 04:20:20.000000000 +0000 @@ -32,16 +32,17 @@ if in_container: for container in container_names: - self.get_container_logs(container) + self.add_container_logs(container) self.add_cmd_output( - self.fmt_container_cmd(container, 'rabbitmqctl report'), + 'rabbitmqctl report', + container=container, foreground=True ) self.add_cmd_output( - self.fmt_container_cmd( - container, "rabbitmqctl eval " - "'rabbit_diagnostics:maybe_stuck().'"), - foreground=True, timeout=10 + "rabbitmqctl eval 'rabbit_diagnostics:maybe_stuck().'", + container=container, + foreground=True, + timeout=10 ) else: self.add_cmd_output("rabbitmqctl report") @@ -59,6 +60,13 @@ "/var/log/rabbitmq/*", ]) + # Crash dump can be large in some situation but it is useful to + # investigate why rabbitmq crashes. So capture the file without + # sizelimit + self.add_copy_spec([ + "/var/log/containers/rabbitmq/erl_crash.dump" + ], sizelimit=0) + def postproc(self): self.do_file_sub("/etc/rabbitmq/rabbitmq.conf", r"(\s*default_pass\s*,\s*)\S+", r"\1<<***>>},") diff -Nru sosreport-4.2/sos/report/plugins/rhui.py sosreport-4.3/sos/report/plugins/rhui.py --- sosreport-4.2/sos/report/plugins/rhui.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/rhui.py 2022-02-15 04:20:20.000000000 +0000 @@ -27,6 +27,7 @@ "/var/log/rhui-subscription-sync.log", "/var/cache/rhui/*", "/root/.rhui/*", + "/var/log/rhui/*", ]) # skip collecting certificate keys self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) diff -Nru sosreport-4.2/sos/report/plugins/rpmostree.py sosreport-4.3/sos/report/plugins/rpmostree.py --- sosreport-4.2/sos/report/plugins/rpmostree.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/rpmostree.py 2022-02-15 04:20:20.000000000 +0000 @@ -22,7 +22,8 @@ self.add_copy_spec('/etc/ostree/remotes.d/') subcmds = [ - 'status', + 'status -v', + 'kargs', 'db list', 'db diff', '--version' diff -Nru sosreport-4.2/sos/report/plugins/rpm.py sosreport-4.3/sos/report/plugins/rpm.py --- sosreport-4.2/sos/report/plugins/rpm.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/rpm.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Rpm(Plugin, RedHatPlugin): @@ -16,10 +16,12 @@ plugin_name = 'rpm' profiles = ('system', 'packagemanager') - option_list = [("rpmq", "queries for package information via rpm -q", - "fast", True), - ("rpmva", "runs a verify on all packages", "slow", False), - ("rpmdb", "collect /var/lib/rpm", "slow", False)] + option_list = [ + PluginOpt('rpmq', default=True, + desc='query package information with rpm -q'), + PluginOpt('rpmva', default=False, desc='verify all packages'), + PluginOpt('rpmdb', default=False, desc='collect /var/lib/rpm') + ] verify_packages = ('rpm',) diff -Nru sosreport-4.2/sos/report/plugins/runc.py sosreport-4.3/sos/report/plugins/runc.py --- sosreport-4.2/sos/report/plugins/runc.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/runc.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,37 +0,0 @@ -# Copyright (C) 2017 Red Hat, Inc. Jake Hunsaker - -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from sos.report.plugins import Plugin, RedHatPlugin - - -class Runc(Plugin): - - short_desc = 'runC container runtime' - plugin_name = 'runc' - profiles = ('container',) - - def setup(self): - - self.add_cmd_output('runc list') - - cons = self.exec_cmd('runc list -q') - conlist = [c for c in cons['output'].splitlines()] - for con in conlist: - self.add_cmd_output('runc ps %s' % con) - self.add_cmd_output('runc state %s' % con) - self.add_cmd_output('runc events --stats %s' % con) - - -class RedHatRunc(Runc, RedHatPlugin): - - packages = ('runc', ) - - def setup(self): - super(RedHatRunc, self).setup() diff -Nru sosreport-4.2/sos/report/plugins/sapnw.py sosreport-4.3/sos/report/plugins/sapnw.py --- sosreport-4.2/sos/report/plugins/sapnw.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/sapnw.py 2022-02-15 04:20:20.000000000 +0000 @@ -38,6 +38,8 @@ if ("DAA" not in inst_line and not inst_line.startswith("No instances found")): fields = inst_line.strip().split() + if len(fields) < 8: + continue sid = fields[3] inst = fields[5] vhost = fields[7] diff -Nru sosreport-4.2/sos/report/plugins/sar.py sosreport-4.3/sos/report/plugins/sar.py --- sosreport-4.2/sos/report/plugins/sar.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/sar.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,8 +6,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin -import os +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) import re @@ -20,11 +20,13 @@ packages = ('sysstat',) sa_path = '/var/log/sa' - option_list = [("all_sar", "gather all system activity records", - "", False)] + option_list = [ + PluginOpt('all_sar', default=False, + desc="gather all system activity records") + ] def setup(self): - self.add_copy_spec(os.path.join(self.sa_path, '*'), + self.add_copy_spec(self.path_join(self.sa_path, '*'), sizelimit=0 if self.get_option("all_sar") else None, tailit=False) @@ -41,7 +43,7 @@ # as option for sadc for fname in dir_list: if sa_regex.match(fname): - sa_data_path = os.path.join(self.sa_path, fname) + sa_data_path = self.path_join(self.sa_path, fname) sar_filename = 'sar' + fname[2:] if sar_filename not in dir_list: sar_cmd = 'sh -c "sar -A -f %s"' % sa_data_path diff -Nru sosreport-4.2/sos/report/plugins/selinux.py sosreport-4.3/sos/report/plugins/selinux.py --- sosreport-4.2/sos/report/plugins/selinux.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/selinux.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class SELinux(Plugin, RedHatPlugin): @@ -16,8 +16,10 @@ plugin_name = 'selinux' profiles = ('container', 'system', 'security', 'openshift') - option_list = [("fixfiles", 'Print incorrect file context labels', - 'slow', False)] + option_list = [ + PluginOpt('fixfiles', default=False, + desc='collect incorrect file context labels') + ] packages = ('libselinux',) def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/services.py sosreport-4.3/sos/report/plugins/services.py --- sosreport-4.2/sos/report/plugins/services.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/services.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, + UbuntuPlugin, PluginOpt) class Services(Plugin): @@ -16,8 +17,10 @@ plugin_name = "services" profiles = ('system', 'boot') - option_list = [("servicestatus", "get a status of all running services", - "slow", False)] + option_list = [ + PluginOpt('servicestatus', default=False, + desc='collect status of all running services') + ] def setup(self): self.add_copy_spec([ diff -Nru sosreport-4.2/sos/report/plugins/skydive.py sosreport-4.3/sos/report/plugins/skydive.py --- sosreport-4.2/sos/report/plugins/skydive.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/skydive.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt import os @@ -26,9 +26,12 @@ password_warn_text = " (password visible in process listings)" option_list = [ - ("username", "skydive user name", "", ""), - ("password", "skydive password" + password_warn_text, "", ""), - ("analyzer", "skydive analyzer address", "", "") + PluginOpt('username', default='', val_type=str, + desc='skydive username'), + PluginOpt('password', default='', val_type=str, + desc='skydive password' + password_warn_text), + PluginOpt('analyzer', default='', val_type=str, + desc='skydive analyzer address') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/smclient.py sosreport-4.3/sos/report/plugins/smclient.py --- sosreport-4.2/sos/report/plugins/smclient.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/smclient.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class SMcli(Plugin, IndependentPlugin): @@ -19,7 +19,7 @@ packages = ('SMclient',) option_list = [ - ("debug", "capture support debug data", "slow", False), + PluginOpt('debug', default=False, desc='capture support debug data') ] def setup(self): diff -Nru sosreport-4.2/sos/report/plugins/sos_extras.py sosreport-4.3/sos/report/plugins/sos_extras.py --- sosreport-4.2/sos/report/plugins/sos_extras.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/sos_extras.py 2022-02-15 04:20:20.000000000 +0000 @@ -58,7 +58,7 @@ for path, dirlist, filelist in os.walk(self.extras_dir): for f in filelist: - _file = os.path.join(path, f) + _file = self.path_join(path, f) self._log_warn("Collecting data from extras file %s" % _file) try: for line in open(_file).read().splitlines(): diff -Nru sosreport-4.2/sos/report/plugins/ssh.py sosreport-4.3/sos/report/plugins/ssh.py --- sosreport-4.2/sos/report/plugins/ssh.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/ssh.py 2022-02-15 04:20:20.000000000 +0000 @@ -33,11 +33,15 @@ # Include main config files self.add_copy_spec(sshcfgs) + self.included_configs(sshcfgs) + self.user_ssh_files_permissions() + + def included_configs(self, sshcfgs): # Read configs for any includes and copy those try: for sshcfg in sshcfgs: tag = sshcfg.split('/')[-1] - with open(sshcfg, 'r') as cfgfile: + with open(self.path_join(sshcfg), 'r') as cfgfile: for line in cfgfile: # skip empty lines and comments if len(line.split()) == 0 or line.startswith('#'): @@ -49,5 +53,33 @@ except Exception: pass + def user_ssh_files_permissions(self): + """ + Iterate over .ssh folders in user homes to see their permissions. + + Bad permissions can prevent SSH from allowing access to given user. + """ + users_data = self.exec_cmd('getent passwd') + + if users_data['status']: + # If getent fails, fallback to just reading /etc/passwd + try: + with open(self.path_join('/etc/passwd')) as passwd_file: + users_data_lines = passwd_file.readlines() + except Exception: + # If we can't read /etc/passwd, then there's something wrong. + self._log_error("Couldn't read /etc/passwd") + return + else: + users_data_lines = users_data['output'].splitlines() + + # Read the home paths of users in the system and check the ~/.ssh dirs + for usr_line in users_data_lines: + try: + home_dir = self.path_join(usr_line.split(':')[5], '.ssh') + if self.path_isdir(home_dir): + self.add_cmd_output('ls -laZ {}'.format(home_dir)) + except IndexError: + pass # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/storcli.py sosreport-4.3/sos/report/plugins/storcli.py --- sosreport-4.2/sos/report/plugins/storcli.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/storcli.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class StorCLI(Plugin, IndependentPlugin): @@ -18,7 +18,7 @@ packages = ('storcli',) option_list = [ - ("json", "collect data in JSON format", "fast", False) + PluginOpt('json', default=False, desc='collect data in JSON format') ] def setup(self): @@ -27,7 +27,6 @@ 'show ctrlcount', '/call show AliLog', '/call show all', - '/call show events', '/call show termlog', '/call/bbu show all', '/call/cv show all', @@ -48,4 +47,12 @@ suggest_filename="storcli64_%s%s" % (subcmd, json), runat=logpath) + # /call show events need 'file=' option to get adapter info like below + # "Adapter: # - Number of Events: xxx". + subcmd = '/call show events' + self.add_cmd_output( + "%s %s file=/dev/stdout%s" % (cmd, subcmd, json), + suggest_filename="storcli64_%s%s" % (subcmd, json), + runat=logpath) + # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/subscription_manager.py sosreport-4.3/sos/report/plugins/subscription_manager.py --- sosreport-4.2/sos/report/plugins/subscription_manager.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/subscription_manager.py 2022-02-15 04:20:20.000000000 +0000 @@ -65,7 +65,8 @@ "subscription-manager identity", "subscription-manager release --show", "subscription-manager release --list", - "syspurpose show" + "syspurpose show", + "subscription-manager syspurpose --show", ], cmd_as_tag=True) self.add_cmd_output("rhsm-debug system --sos --no-archive " "--no-subscriptions --destination %s" diff -Nru sosreport-4.2/sos/report/plugins/systemd.py sosreport-4.3/sos/report/plugins/systemd.py --- sosreport-4.2/sos/report/plugins/systemd.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/systemd.py 2022-02-15 04:20:20.000000000 +0000 @@ -39,6 +39,7 @@ # status --all mostly seems to cover the others. "systemctl list-units", "systemctl list-units --failed", + "systemctl list-units --all", "systemctl list-unit-files", "systemctl list-jobs", "systemctl list-dependencies", diff -Nru sosreport-4.2/sos/report/plugins/system.py sosreport-4.3/sos/report/plugins/system.py --- sosreport-4.2/sos/report/plugins/system.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/system.py 2022-02-15 04:20:20.000000000 +0000 @@ -33,5 +33,12 @@ "/proc/sys/net/ipv6/neigh/*/base_reachable_time" ]) + # collect glibc tuning decisions + self.add_cmd_output([ + "ld.so --help", + "ld.so --list-diagnostics", + "ld.so --list-tunables" + ]) + # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/udisks.py sosreport-4.3/sos/report/plugins/udisks.py --- sosreport-4.2/sos/report/plugins/udisks.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/udisks.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,30 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin + + +class Udisks(Plugin, IndependentPlugin): + + short_desc = 'udisks disk manager' + + plugin_name = 'udisks' + profiles = ('system', 'hardware') + commands = ('udisksctl',) + + def setup(self): + self.add_copy_spec([ + "/etc/udisks2/", + ]) + + self.add_cmd_output([ + "udisksctl status", + "udisksctl dump", + ]) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/unbound.py sosreport-4.3/sos/report/plugins/unbound.py --- sosreport-4.2/sos/report/plugins/unbound.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/unbound.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,30 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin + + +class Unbound(Plugin, IndependentPlugin): + + short_desc = 'Unbound DNS resolver' + + plugin_name = 'unbound' + profiles = ('system', 'services', 'network') + packages = ('unbound', 'unbound-libs') + + def setup(self): + self.add_copy_spec([ + "/etc/sysconfig/unbound", + "/etc/unbound/unbound.conf", + "/usr/lib/tmpfiles.d/unbound.conf", + "/etc/unbound/conf.d/", + "/etc/unbound/local.d/", + ]) + + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/unpackaged.py sosreport-4.3/sos/report/plugins/unpackaged.py --- sosreport-4.2/sos/report/plugins/unpackaged.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/unpackaged.py 2022-02-15 04:20:20.000000000 +0000 @@ -10,6 +10,7 @@ import os import stat +from pathlib import Path class Unpackaged(Plugin, RedHatPlugin): @@ -26,7 +27,7 @@ return os.environ['PATH'].split(':') def all_files_system(path, exclude=None): - """Retrun a list of all files present on the system, excluding + """Return a list of all files present on the system, excluding any directories listed in `exclude`. :param path: the starting path @@ -39,16 +40,16 @@ for e in exclude: dirs[:] = [d for d in dirs if d not in e] for name in files: - path = os.path.join(root, name) + path = self.path_join(root, name) try: - while stat.S_ISLNK(os.lstat(path).st_mode): - path = os.path.abspath(os.readlink(path)) + if stat.S_ISLNK(os.lstat(path).st_mode): + path = Path(path).resolve() except Exception: continue file_list.append(os.path.realpath(path)) for name in dirs: file_list.append(os.path.realpath( - os.path.join(root, name))) + self.path_join(root, name))) return file_list @@ -57,25 +58,30 @@ """ expanded = [] for f in files: - if self.path_islink(f): - expanded.append("{} -> {}".format(f, os.readlink(f))) + fp = self.path_join(f) + if self.path_islink(fp): + expanded.append("{} -> {}".format(fp, os.readlink(fp))) else: - expanded.append(f) + expanded.append(fp) return expanded # Check command predicate to avoid costly processing if not self.test_predicate(cmd=True): return + paths = get_env_path_list() all_fsystem = [] - all_frpm = set(os.path.realpath(x) - for x in self.policy.mangle_package_path( - self.policy.package_manager.all_files())) + all_frpm = set( + os.path.realpath(x) for x in self.policy.mangle_package_path( + self.policy.package_manager.all_files() + ) if any([x.startswith(p) for p in paths]) + ) - for d in get_env_path_list(): + for d in paths: all_fsystem += all_files_system(d) not_packaged = [x for x in all_fsystem if x not in all_frpm] not_packaged_expanded = format_output(not_packaged) - self.add_string_as_file('\n'.join(not_packaged_expanded), 'unpackaged') + self.add_string_as_file('\n'.join(not_packaged_expanded), 'unpackaged', + plug_dir=True) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/vdsm.py sosreport-4.3/sos/report/plugins/vdsm.py --- sosreport-4.2/sos/report/plugins/vdsm.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/vdsm.py 2022-02-15 04:20:20.000000000 +0000 @@ -63,6 +63,7 @@ self.add_forbidden_path('/etc/pki/vdsm/keys') self.add_forbidden_path('/etc/pki/vdsm/*/*-key.*') self.add_forbidden_path('/etc/pki/libvirt/private') + self.add_forbidden_path('/var/lib/vdsm/storage/transient_disks') self.add_service_status(['vdsmd', 'supervdsmd']) diff -Nru sosreport-4.2/sos/report/plugins/veritas.py sosreport-4.3/sos/report/plugins/veritas.py --- sosreport-4.2/sos/report/plugins/veritas.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/veritas.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt class Veritas(Plugin, RedHatPlugin): @@ -18,8 +18,10 @@ # Information about VRTSexplorer obtained from # http://seer.entsupport.symantec.com/docs/243150.htm - option_list = [("script", "Define VRTSexplorer script path", "", - "/opt/VRTSspt/VRTSexplorer")] + option_list = [ + PluginOpt('script', default='/opt/VRTSspt/VRTSexplorer', + desc='Path to VRTSexploer script') + ] def check_enabled(self): return self.path_isfile(self.get_option("script")) diff -Nru sosreport-4.2/sos/report/plugins/virsh.py sosreport-4.3/sos/report/plugins/virsh.py --- sosreport-4.2/sos/report/plugins/virsh.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/virsh.py 2022-02-15 04:20:20.000000000 +0000 @@ -39,26 +39,34 @@ ] for subcmd in subcmds: - self.add_cmd_output('%s %s' % (cmd, subcmd)) + self.add_cmd_output('%s %s' % (cmd, subcmd), foreground=True) # get network, pool and nwfilter elements for k in ['net', 'nwfilter', 'pool']: - k_list = self.collect_cmd_output('%s %s-list' % (cmd, k)) + k_list = self.collect_cmd_output('%s %s-list' % (cmd, k), + foreground=True) if k_list['status'] == 0: k_lines = k_list['output'].splitlines() # the 'Name' column position changes between virsh cmds - pos = k_lines[0].split().index('Name') + # catch the rare exceptions when 'Name' is not found + try: + pos = k_lines[0].split().index('Name') + except Exception: + continue for j in filter(lambda x: x, k_lines[2:]): n = j.split()[pos] - self.add_cmd_output('%s %s-dumpxml %s' % (cmd, k, n)) + self.add_cmd_output('%s %s-dumpxml %s' % (cmd, k, n), + foreground=True) # cycle through the VMs/domains list, ignore 2 header lines and latest # empty line, and dumpxml domain name in 2nd column - domains_output = self.exec_cmd('%s list --all' % cmd) + domains_output = self.exec_cmd('%s list --all' % cmd, foreground=True) if domains_output['status'] == 0: domains_lines = domains_output['output'].splitlines()[2:] for domain in filter(lambda x: x, domains_lines): d = domain.split()[1] for x in ['dumpxml', 'dominfo', 'domblklist']: - self.add_cmd_output('%s %s %s' % (cmd, x, d)) + self.add_cmd_output('%s %s %s' % (cmd, x, d), + foreground=True) + # vim: et ts=4 sw=4 diff -Nru sosreport-4.2/sos/report/plugins/vmware.py sosreport-4.3/sos/report/plugins/vmware.py --- sosreport-4.2/sos/report/plugins/vmware.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/vmware.py 2022-02-15 04:20:20.000000000 +0000 @@ -37,7 +37,8 @@ self.add_cmd_output([ "vmware-checkvm", "vmware-toolbox-cmd device list", - "vmware-toolbox-cmd -v" + "vmware-toolbox-cmd -v", + "vmware-toolbox-cmd timesync status" ]) stats = self.exec_cmd("vmware-toolbox-cmd stat raw") diff -Nru sosreport-4.2/sos/report/plugins/watchdog.py sosreport-4.3/sos/report/plugins/watchdog.py --- sosreport-4.2/sos/report/plugins/watchdog.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/watchdog.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,10 +8,9 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt from glob import glob -import os class Watchdog(Plugin, RedHatPlugin): @@ -22,7 +21,8 @@ packages = ('watchdog',) option_list = [ - ('conf_file', 'watchdog config file', 'fast', '/etc/watchdog.conf'), + PluginOpt('conf_file', default='/etc/watchdog.conf', + desc='watchdog config file') ] def get_log_dir(self, conf_file): @@ -55,8 +55,8 @@ Collect configuration files, custom executables for test-binary and repair-binary, and stdout/stderr logs. """ - conf_file = self.get_option('conf_file') - log_dir = '/var/log/watchdog' + conf_file = self.path_join(self.get_option('conf_file')) + log_dir = self.path_join('/var/log/watchdog') # Get service configuration and sysconfig files self.add_copy_spec([ @@ -79,15 +79,15 @@ self._log_warn("Could not read %s: %s" % (conf_file, ex)) if self.get_option('all_logs'): - log_files = glob(os.path.join(log_dir, '*')) + log_files = glob(self.path_join(log_dir, '*')) else: - log_files = (glob(os.path.join(log_dir, '*.stdout')) + - glob(os.path.join(log_dir, '*.stderr'))) + log_files = (glob(self.path_join(log_dir, '*.stdout')) + + glob(self.path_join(log_dir, '*.stderr'))) self.add_copy_spec(log_files) # Get output of "wdctl " for each /dev/watchdog* - for dev in glob('/dev/watchdog*'): + for dev in glob(self.path_join('/dev/watchdog*')): self.add_cmd_output("wdctl %s" % dev) # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/sos/report/plugins/yum.py sosreport-4.3/sos/report/plugins/yum.py --- sosreport-4.2/sos/report/plugins/yum.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/report/plugins/yum.py 2022-02-15 04:20:20.000000000 +0000 @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin +from sos.report.plugins import Plugin, RedHatPlugin, PluginOpt import os YUM_PLUGIN_PATH = "/usr/lib/yum-plugins/" @@ -24,9 +24,10 @@ verify_packages = ('yum',) option_list = [ - ("yumlist", "list repositories and packages", "slow", False), - ("yumdebug", "gather yum debugging data", "slow", False), - ("yum-history-info", "gather yum history info", "slow", False), + PluginOpt('yumlist', default=False, desc='list repos and packages'), + PluginOpt('yumdebug', default=False, desc='collect yum debug data'), + PluginOpt('yum-history-info', default=False, + desc='collect yum history info for all transactions') ] def setup(self): @@ -60,7 +61,7 @@ if not p.endswith(".py"): continue plugins = plugins + " " if len(plugins) else "" - plugins = plugins + os.path.join(YUM_PLUGIN_PATH, p) + plugins = plugins + self.path_join(YUM_PLUGIN_PATH, p) if len(plugins): self.add_cmd_output("rpm -qf %s" % plugins, suggest_filename="plugin-packages") @@ -68,7 +69,8 @@ os.path.basename(p)[:-3] for p in plugins.split() ] plugnames = "%s\n" % "\n".join(plugnames) - self.add_string_as_file(plugnames, "plugin-names") + self.add_string_as_file(plugnames, "plugin-names", + plug_dir=True) self.add_copy_spec("/etc/yum/pluginconf.d") diff -Nru sosreport-4.2/sos/utilities.py sosreport-4.3/sos/utilities.py --- sosreport-4.2/sos/utilities.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos/utilities.py 2022-02-15 04:20:20.000000000 +0000 @@ -96,17 +96,22 @@ return matches -def is_executable(command): +def is_executable(command, sysroot=None): """Returns if a command matches an executable on the PATH""" paths = os.environ.get("PATH", "").split(os.path.pathsep) candidates = [command] + [os.path.join(p, command) for p in paths] + if sysroot: + candidates += [ + os.path.join(sysroot, c.lstrip('/')) for c in candidates + ] return any(os.access(path, os.X_OK) for path in candidates) def sos_get_command_output(command, timeout=TIMEOUT_DEFAULT, stderr=False, chroot=None, chdir=None, env=None, foreground=False, - binary=False, sizelimit=None, poller=None): + binary=False, sizelimit=None, poller=None, + to_file=False): """Execute a command and return a dictionary of status and output, optionally changing root or current working directory before executing command. @@ -115,11 +120,17 @@ # closure are caught in the parent (chroot and chdir are bound from # the enclosing scope). def _child_prep_fn(): - if (chroot): + if chroot and chroot != '/': os.chroot(chroot) if (chdir): os.chdir(chdir) + def _check_poller(proc): + if poller(): + proc.terminate() + raise SoSTimeoutError + time.sleep(0.01) + cmd_env = os.environ.copy() # ensure consistent locale for collected command output cmd_env['LC_ALL'] = 'C' @@ -139,32 +150,57 @@ ) args = shlex.split(command) - # Expand arguments that are wildcard paths. + # Expand arguments that are wildcard root paths. expanded_args = [] for arg in args: - expanded_arg = glob.glob(arg) - if expanded_arg: - expanded_args.extend(expanded_arg) + if arg.startswith("/") and "*" in arg: + expanded_arg = glob.glob(arg) + if expanded_arg: + expanded_args.extend(expanded_arg) + else: + expanded_args.append(arg) else: expanded_args.append(arg) + if to_file: + _output = open(to_file, 'w') + else: + _output = PIPE try: - p = Popen(expanded_args, shell=False, stdout=PIPE, + p = Popen(expanded_args, shell=False, stdout=_output, stderr=STDOUT if stderr else PIPE, bufsize=-1, env=cmd_env, close_fds=True, preexec_fn=_child_prep_fn) - reader = AsyncReader(p.stdout, sizelimit, binary) + if not to_file: + reader = AsyncReader(p.stdout, sizelimit, binary) + else: + reader = FakeReader(p, binary) + if poller: while reader.running: - if poller(): - p.terminate() - raise SoSTimeoutError - time.sleep(0.01) - stdout = reader.get_contents() - truncated = reader.is_full + _check_poller(p) + else: + try: + # override timeout=0 to timeout=None, as Popen will treat the + # former as a literal 0-second timeout + p.wait(timeout if timeout else None) + except Exception: + p.terminate() + _output.close() + # until we separate timeouts from the `timeout` command + # handle per-cmd timeouts via Plugin status checks + return {'status': 124, 'output': reader.get_contents(), + 'truncated': reader.is_full} + if to_file: + _output.close() + + # wait for Popen to set the returncode while p.poll() is None: pass + stdout = reader.get_contents() + truncated = reader.is_full + except OSError as e: if e.errno == errno.ENOENT: return {'status': 127, 'output': "", 'truncated': ''} @@ -216,8 +252,9 @@ def _os_wrapper(path, sysroot, method, module=os.path): - if sysroot not in [None, '/']: - path = os.path.join(sysroot, path.lstrip('/')) + if sysroot and sysroot != os.sep: + if not path.startswith(sysroot): + path = os.path.join(sysroot, path.lstrip('/')) _meth = getattr(module, method) return _meth(path) @@ -242,6 +279,47 @@ return _os_wrapper(path, sysroot, 'listdir', os) +def path_join(path, *p, sysroot=os.sep): + if sysroot and not path.startswith(sysroot): + path = os.path.join(sysroot, path.lstrip(os.sep)) + return os.path.join(path, *p) + + +def bold(text): + """Helper to make text bold in console output, without pulling in + dependencies to the project unneccessarily. + + :param text: The text to make bold + :type text: ``str`` + + :returns: The text wrapped in the ASCII codes to display as bold + :rtype: ``str`` + """ + return '\033[1m' + text + '\033[0m' + + +class FakeReader(): + """Used as a replacement AsyncReader for when we are writing directly to + disk, and allows us to keep more simplified flows for executing, + monitoring, and collecting command output. + """ + + def __init__(self, process, binary): + self.process = process + self.binary = binary + + @property + def is_full(self): + return False + + def get_contents(self): + return '' if not self.binary else b'' + + @property + def running(self): + return self.process.poll() is None + + class AsyncReader(threading.Thread): """Used to limit command output to a given size without deadlocking sos. diff -Nru sosreport-4.2/sos.conf sosreport-4.3/sos.conf --- sosreport-4.2/sos.conf 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos.conf 2022-02-15 04:20:20.000000000 +0000 @@ -19,7 +19,7 @@ # Options that will apply to any `sos collect` run should be listed here. # Note that the option names *must* be the long-form name as seen in --help # output. Use a comma for list delimitations -#master = myhost.example.com +#primary = myhost.example.com #ssh-key = /home/user/.ssh/mykey #password = true diff -Nru sosreport-4.2/sos.spec sosreport-4.3/sos.spec --- sosreport-4.2/sos.spec 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/sos.spec 2022-02-15 04:20:20.000000000 +0000 @@ -2,7 +2,7 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos -Version: 4.2 +Version: 4.3 Release: 1%{?dist} Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{name}-%{version}.tar.gz diff -Nru sosreport-4.2/tests/cleaner_tests/existing_archive.py sosreport-4.3/tests/cleaner_tests/existing_archive.py --- sosreport-4.2/tests/cleaner_tests/existing_archive.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/cleaner_tests/existing_archive.py 2022-02-15 04:20:20.000000000 +0000 @@ -28,6 +28,13 @@ def test_obfuscation_log_created(self): self.assertFileExists(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE)) + def test_archive_type_correct(self): + with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log: + for line in log: + if "Loaded %s" % ARCHIVE in line: + assert 'as type sos report archive' in line, "Incorrect archive type detected: %s" % line + break + def test_from_cmdline_logged(self): with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log: for line in log: diff -Nru sosreport-4.2/tests/cleaner_tests/full_report_run.py sosreport-4.3/tests/cleaner_tests/full_report_run.py --- sosreport-4.2/tests/cleaner_tests/full_report_run.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/cleaner_tests/full_report_run.py 2022-02-15 04:20:20.000000000 +0000 @@ -26,6 +26,24 @@ # replace with an empty placeholder, make sure that this test case is not # influenced by previous clean runs files = ['/etc/sos/cleaner/default_mapping'] + packages = { + 'rhel': ['python3-systemd'], + 'ubuntu': ['python3-systemd'] + } + + def pre_sos_setup(self): + # ensure that case-insensitive matching of FQDNs and shortnames work + from systemd import journal + from socket import gethostname + host = gethostname() + short = host.split('.')[0] + sosfd = journal.stream('sos-testing') + sosfd.write( + "This is a test line from sos clean testing. The hostname %s " + "should not appear, nor should %s in an obfuscated archive. The " + "shortnames of %s and %s should also not appear." + % (host.lower(), host.upper(), short.lower(), short.upper()) + ) def test_private_map_was_generated(self): self.assertOutputContains('A mapping of obfuscated elements is available at') @@ -35,10 +53,14 @@ def test_tarball_named_obfuscated(self): self.assertTrue('obfuscated' in self.archive) + def test_archive_type_correct(self): + self.assertSosLogContains('Loaded .* as type sos report directory') + def test_hostname_not_in_any_file(self): host = self.sysinfo['pre']['networking']['hostname'] + short = host.split('.')[0] # much faster to just use grep here - content = self.grep_for_content(host) + content = self.grep_for_content(host) + self.grep_for_content(short) if not content: assert True else: diff -Nru sosreport-4.2/tests/cleaner_tests/report_with_mask.py sosreport-4.3/tests/cleaner_tests/report_with_mask.py --- sosreport-4.2/tests/cleaner_tests/report_with_mask.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/cleaner_tests/report_with_mask.py 2022-02-15 04:20:20.000000000 +0000 @@ -31,6 +31,9 @@ def test_tarball_named_obfuscated(self): self.assertTrue('obfuscated' in self.archive) + def test_archive_type_correct(self): + self.assertSosLogContains('Loaded .* as type sos report directory') + def test_localhost_was_obfuscated(self): self.assertFileHasContent('/etc/hostname', 'host0') diff -Nru sosreport-4.2/tests/report_tests/basic_report_tests.py sosreport-4.3/tests/report_tests/basic_report_tests.py --- sosreport-4.2/tests/report_tests/basic_report_tests.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/report_tests/basic_report_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -15,7 +15,7 @@ :avocado: tags=stageone """ - sos_cmd = '-v --label thisismylabel' + sos_cmd = '--label thisismylabel' def test_debug_in_logs_verbose(self): self.assertSosLogContains('DEBUG') diff -Nru sosreport-4.2/tests/report_tests/options_tests.py sosreport-4.3/tests/report_tests/options_tests.py --- sosreport-4.2/tests/report_tests/options_tests.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/tests/report_tests/options_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,45 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos_tests import StageTwoReportTest + + +class OptionsFromConfigTest(StageTwoReportTest): + """Ensure that we handle options specified in sos.conf properly + + :avocado: tags=stagetwo + """ + + files = [('/etc/sos/options_tests_sos.conf', '/etc/sos/sos.conf')] + sos_cmd = '-v ' + + def test_case_id_from_config(self): + self.assertTrue('8675309' in self.archive) + + def test_plugins_skipped_from_config(self): + self.assertPluginNotIncluded(['networking', 'logs']) + + def test_plugopts_logged_from_config(self): + self.assertSosLogContains( + "Set kernel plugin option to \(name=with-timer, desc='gather /proc/timer\* statistics', value=True, default=False\)" + ) + self.assertSosLogContains( + "Set kernel plugin option to \(name=trace, desc='gather /sys/kernel/debug/tracing/trace file', value=True, default=False\)" + ) + + def test_disabled_plugopts_not_loaded(self): + self.assertSosLogNotContains("Set networking plugin option to") + + def test_plugopts_actually_set(self): + self.assertFileCollected('sys/kernel/debug/tracing/trace') + + def test_effective_options_logged_correctly(self): + self.assertSosLogContains( + "effective options now: --batch --case-id 8675309 --plugopts kernel.with-timer=on,kernel.trace=yes --skip-plugins networking,logs" + ) diff -Nru sosreport-4.2/tests/report_tests/plugin_tests/logs.py sosreport-4.3/tests/report_tests/plugin_tests/logs.py --- sosreport-4.2/tests/report_tests/plugin_tests/logs.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/report_tests/plugin_tests/logs.py 2022-02-15 04:20:20.000000000 +0000 @@ -41,6 +41,7 @@ """ sos_cmd = '-o logs' + sos_timeout = 500 packages = { 'rhel': ['python3-systemd'], 'ubuntu': ['python3-systemd'] @@ -74,3 +75,9 @@ self.assertFileExists(tailed) journ = self.get_name_in_archive('sos_commands/logs/journalctl_--no-pager') assert os.path.islink(journ), "Journal in sos_commands/logs is not a symlink" + + def test_string_not_in_manifest(self): + # we don't want truncated collections appearing in the strings section + # of the manifest for the plugin + manifest = self.get_plugin_manifest('logs') + self.assertFalse(manifest['strings']) diff -Nru sosreport-4.2/tests/report_tests/plugin_tests/string_collection_tests.py sosreport-4.3/tests/report_tests/plugin_tests/string_collection_tests.py --- sosreport-4.2/tests/report_tests/plugin_tests/string_collection_tests.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/tests/report_tests/plugin_tests/string_collection_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,37 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + + +from sos_tests import StageOneReportTest + + +class CollectStringTest(StageOneReportTest): + """Test to ensure that add_string_as_file() is working for plugins that + directly call it as part of their collections + + :avocado: tags=stageone + """ + + sos_cmd = '-o unpackaged,python -k python.hashes' + # unpackaged is only a RedHatPlugin + redhat_only = True + + def test_unpackaged_list_collected(self): + self.assertFileCollected('sos_commands/unpackaged/unpackaged') + + def test_python_hashes_collected(self): + self.assertFileCollected('sos_commands/python/digests.json') + + def test_no_strings_dir(self): + self.assertFileNotCollected('sos_strings/') + + def test_manifest_strings_correct(self): + pkgman = self.get_plugin_manifest('unpackaged') + self.assertTrue(pkgman['strings']['unpackaged']) + pyman = self.get_plugin_manifest('python') + self.assertTrue(pyman['strings']['digests_json']) diff -Nru sosreport-4.2/tests/sos_tests.py sosreport-4.3/tests/sos_tests.py --- sosreport-4.2/tests/sos_tests.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/sos_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -68,6 +68,7 @@ sos_timeout = 300 redhat_only = False ubuntu_only = False + end_of_test_case = False @property def klass_name(self): @@ -120,7 +121,8 @@ """ exec_cmd = self._generate_sos_command() try: - self.cmd_output = process.run(exec_cmd, timeout=self.sos_timeout) + self.cmd_output = process.run(exec_cmd, timeout=self.sos_timeout, + env={'SOS_TEST_LOGS': 'keep'}) except Exception as err: if not hasattr(err, 'result'): # can't inspect the exception raised, just bail out @@ -255,6 +257,23 @@ break self.sysinfo = self.get_sysinfo() + def tearDown(self): + """If a test being run is the last one defined for a test case, then + we should remove the extracted tarball directory so that we can upload + a reasonably sized artifact to the GCE storage bucket if any tests + fail during the test execution. + + The use of `end_of_test_case` is a bit wonky because we don't have a + different/more reliable way to identify that a test class has completed + all tests - mainly because avocado re-initializes the entire class + for each `test_*` method defined within it. + """ + if self.end_of_test_case: + # remove the extracted directory only if we have the tarball + if self.archive and os.path.exists(self.archive): + if os.path.exists(self.archive_path): + shutil.rmtree(self.archive_path) + def setup_mocking(self): """Since we need to use setUp() in our overrides of avocado.Test, provide an alternate method for test cases that subclass BaseSoSTest @@ -391,7 +410,7 @@ return os.path.join(self.tmpdir, "sosreport-%s" % self.__class__.__name__) def _generate_sos_command(self): - return "%s %s --batch --tmp-dir %s %s" % (SOS_BIN, self.sos_component, self.tmpdir, self.sos_cmd) + return "%s %s -v --batch --tmp-dir %s %s" % (SOS_BIN, self.sos_component, self.tmpdir, self.sos_cmd) def _execute_sos_cmd(self): super(BaseSoSReportTest, self)._execute_sos_cmd() @@ -654,10 +673,6 @@ def test_html_reports_created(self): self.assertFileCollected('sos_reports/sos.html') - def test_no_exceptions_during_execution(self): - self.assertSosLogNotContains('caught exception in plugin') - self.assertFileGlobNotInArchive('sos_logs/*-plugin-errors.txt') - def test_no_ip_changes(self): # I.E. make sure we didn't cause any NIC flaps that for some reason # resulted in a new primary IP address. TODO: build this out to make @@ -665,6 +680,11 @@ self.assertEqual(self.sysinfo['pre']['networking']['ip_addr'], self.sysinfo['post']['networking']['ip_addr']) + def test_no_exceptions_during_execution(self): + self.end_of_test_case = True + self.assertSosLogNotContains('caught exception in plugin') + self.assertFileGlobNotInArchive('sos_logs/*-plugin-errors.txt') + class StageTwoReportTest(BaseSoSReportTest): """This is the testing class to subclass when light mocking is needed to @@ -685,7 +705,10 @@ files - a list containing the files to drop on the test system's real filesystem. Mocked files should be placed in the same locations - under tests/test_data + under tests/test_data. If list items are tuples, then the tuple + elements are (source_path, dest_path), which will allow the + project to store multiple versions of files in the tree without + interfering with other tests packages - a dict where the keys are the distribution names (e.g. 'rhel', 'ubuntu') and the values are the package names optionally with @@ -733,6 +756,7 @@ def tearDown(self): if self.end_of_test_case: self.teardown_mocking() + super(StageTwoReportTest, self).tearDown() def teardown_mocking(self): """Undo any and all mocked setup that we did for tests @@ -807,6 +831,27 @@ for pkg in pkgs: self.sm.remove(pkg) + def _copy_test_file(self, src, dest=None): + """Helper to copy files from tests/test_data to relevant locations on + the test system. If ``dest`` is provided, use that as the destination + filename instead of using the ``src`` name + """ + + if dest is None: + dest = src + dir_added = False + if os.path.exists(dest): + os.rename(dest, dest + '.sostesting') + _dir = os.path.split(src)[0] + if not os.path.exists(_dir): + os.makedirs(_dir) + self._created_files.append(_dir) + dir_added = True + _test_file = os.path.join(SOS_TEST_DIR, 'test_data', src.lstrip('/')) + shutil.copy(_test_file, dest) + if not dir_added: + self._created_files.append(dest) + def setup_mocked_files(self): """Place any requested files from under tests/test_data into "proper" locations on the test system's filesystem. @@ -816,18 +861,10 @@ test(s) have run. """ for mfile in self.files: - dir_added = False - if os.path.exists(mfile): - os.rename(mfile, mfile + '.sostesting') - _dir = os.path.split(mfile)[0] - if not os.path.exists(_dir): - os.makedirs(_dir) - self._created_files.append(_dir) - dir_added = True - _test_file = os.path.join(SOS_TEST_DIR, 'test_data', mfile.lstrip('/')) - shutil.copy(_test_file, mfile) - if not dir_added: - self._created_files.append(mfile) + if isinstance(mfile, tuple): + self._copy_test_file(mfile[0], mfile[1]) + else: + self._copy_test_file(mfile) if self._created_files: self._write_file_to_tmpdir('mocked_files', json.dumps(self._created_files)) diff -Nru sosreport-4.2/tests/test_data/etc/sos/options_tests_sos.conf sosreport-4.3/tests/test_data/etc/sos/options_tests_sos.conf --- sosreport-4.2/tests/test_data/etc/sos/options_tests_sos.conf 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/tests/test_data/etc/sos/options_tests_sos.conf 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,18 @@ +[global] +#verbose = 3 + +[report] +skip-plugins = networking,logs +case-id = 8675309 + +[collect] +#primary = myhost.example.com + +[clean] +#no-update = true + +[plugin_options] +#rpm.rpmva = off +kernel.with-timer = on +kernel.trace = yes +networking.traceroute = yes diff -Nru sosreport-4.2/tests/test_data/etc/sos/sos.conf sosreport-4.3/tests/test_data/etc/sos/sos.conf --- sosreport-4.2/tests/test_data/etc/sos/sos.conf 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/test_data/etc/sos/sos.conf 2022-02-15 04:20:20.000000000 +0000 @@ -5,7 +5,7 @@ #skip-plugins = rpm,selinux,dovecot [collect] -#master = myhost.example.com +#primary = myhost.example.com [clean] keywords = shibboleth diff -Nru sosreport-4.2/tests/unittests/cleaner_tests.py sosreport-4.3/tests/unittests/cleaner_tests.py --- sosreport-4.2/tests/unittests/cleaner_tests.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/unittests/cleaner_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -105,6 +105,7 @@ self.host_parser = SoSHostnameParser(config={}, opt_domains='foobar.com') self.kw_parser = SoSKeywordParser(config={}, keywords=['foobar']) self.kw_parser_none = SoSKeywordParser(config={}) + self.kw_parser.generate_item_regexes() def test_ip_parser_valid_ipv4_line(self): line = 'foobar foo 10.0.0.1/24 barfoo bar' diff -Nru sosreport-4.2/tests/unittests/conformance_tests.py sosreport-4.3/tests/unittests/conformance_tests.py --- sosreport-4.2/tests/unittests/conformance_tests.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/unittests/conformance_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -8,7 +8,7 @@ import unittest -from sos.report.plugins import import_plugin +from sos.report.plugins import import_plugin, PluginOpt from sos.utilities import ImporterHelper @@ -50,7 +50,8 @@ for plug in self.plug_classes: self.assertIsInstance(plug.option_list, list) for opt in plug.option_list: - self.assertIsInstance(opt, tuple) + self.assertIsInstance(opt, PluginOpt) + self.assertFalse(opt.name == 'undefined') def test_plugin_architectures_set_correctly(self): for plug in self.plug_classes: diff -Nru sosreport-4.2/tests/unittests/option_tests.py sosreport-4.3/tests/unittests/option_tests.py --- sosreport-4.2/tests/unittests/option_tests.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/unittests/option_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -7,7 +7,7 @@ # See the LICENSE file in the source distribution for further information. import unittest -from sos.report.plugins import Plugin +from sos.report.plugins import Plugin, PluginOpt from sos.policies.distros import LinuxPolicy from sos.policies.init_systems import InitSystem @@ -21,6 +21,18 @@ skip_files = [] +class MockPlugin(Plugin): + + option_list = [ + PluginOpt('baz', default=False), + PluginOpt('empty', default=None), + PluginOpt('test_option', default='foobar') + ] + + def __init__(self, commons): + super(MockPlugin, self).__init__(commons=commons) + + class GlobalOptionTest(unittest.TestCase): def setUp(self): @@ -30,11 +42,7 @@ 'cmdlineopts': MockOptions(), 'devices': {} } - self.plugin = Plugin(self.commons) - self.plugin.opt_names = ['baz', 'empty', 'test_option'] - self.plugin.opt_parms = [ - {'enabled': False}, {'enabled': None}, {'enabled': 'foobar'} - ] + self.plugin = MockPlugin(self.commons) def test_simple_lookup(self): self.assertEquals(self.plugin.get_option('test_option'), 'foobar') diff -Nru sosreport-4.2/tests/unittests/plugin_tests.py sosreport-4.3/tests/unittests/plugin_tests.py --- sosreport-4.2/tests/unittests/plugin_tests.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/unittests/plugin_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -12,7 +12,7 @@ from io import StringIO -from sos.report.plugins import Plugin, regex_findall, _mangle_command +from sos.report.plugins import Plugin, regex_findall, _mangle_command, PluginOpt from sos.archive import TarFileArchive from sos.policies.distros import LinuxPolicy from sos.policies.init_systems import InitSystem @@ -64,8 +64,10 @@ class MockPlugin(Plugin): - option_list = [("opt", 'an option', 'fast', None), - ("opt2", 'another option', 'fast', False)] + option_list = [ + PluginOpt("opt", default=None, desc='an option', val_type=str), + PluginOpt("opt2", default=False, desc='another option') + ] def setup(self): pass @@ -271,35 +273,6 @@ }) self.assertEquals(p.get_option("opt2", True), False) - def test_get_option_as_list_plugin_option(self): - p = MockPlugin({ - 'sysroot': self.sysroot, - 'policy': LinuxPolicy(init=InitSystem(), probe_runtime=False), - 'cmdlineopts': MockOptions(), - 'devices': {} - }) - p.set_option("opt", "one,two,three") - self.assertEquals(p.get_option_as_list("opt"), ['one', 'two', 'three']) - - def test_get_option_as_list_plugin_option_default(self): - p = MockPlugin({ - 'sysroot': self.sysroot, - 'policy': LinuxPolicy(init=InitSystem(), probe_runtime=False), - 'cmdlineopts': MockOptions(), - 'devices': {} - }) - self.assertEquals(p.get_option_as_list("opt", default=[]), []) - - def test_get_option_as_list_plugin_option_not_list(self): - p = MockPlugin({ - 'sysroot': self.sysroot, - 'policy': LinuxPolicy(init=InitSystem(), probe_runtime=False), - 'cmdlineopts': MockOptions(), - 'devices': {} - }) - p.set_option("opt", "testing") - self.assertEquals(p.get_option_as_list("opt"), ['testing']) - def test_copy_dir(self): self.mp._do_copy_path("tests") self.assertEquals( @@ -366,10 +339,9 @@ self.mp.sysroot = '/' fn = create_file(2) # create 2MB file, consider a context manager self.mp.add_copy_spec(fn, 1) - content, fname = self.mp.copy_strings[0] + content, fname, _tags = self.mp.copy_strings[0] self.assertTrue("tailed" in fname) self.assertTrue("tmp" in fname) - self.assertTrue("/" not in fname) self.assertEquals(1024 * 1024, len(content)) os.unlink(fn) @@ -398,7 +370,7 @@ create_file(2, dir=tmpdir) self.mp.add_copy_spec(tmpdir + "/*", 1) self.assertEquals(len(self.mp.copy_strings), 1) - content, fname = self.mp.copy_strings[0] + content, fname, _tags = self.mp.copy_strings[0] self.assertTrue("tailed" in fname) self.assertEquals(1024 * 1024, len(content)) shutil.rmtree(tmpdir) @@ -457,6 +429,7 @@ "never_copied", r"^(.*)$", "foobar")) def test_no_replacements(self): + self.mp.sysroot = '/' self.mp.add_copy_spec(j("tail_test.txt")) self.mp.collect() replacements = self.mp.do_file_sub( diff -Nru sosreport-4.2/tests/unittests/sosreport_pexpect.py sosreport-4.3/tests/unittests/sosreport_pexpect.py --- sosreport-4.2/tests/unittests/sosreport_pexpect.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/unittests/sosreport_pexpect.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,36 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. -import unittest -import pexpect - -from os import kill -from signal import SIGINT - - -class PexpectTest(unittest.TestCase): - def test_plugins_install(self): - sos = pexpect.spawn('/usr/sbin/sosreport -l') - try: - sos.expect('plugin.*does not install, skipping') - except pexpect.EOF: - pass - else: - self.fail("a plugin does not install or sosreport is too slow") - kill(sos.pid, SIGINT) - - def test_batchmode_removes_questions(self): - sos = pexpect.spawn('/usr/sbin/sosreport --batch') - grp = sos.expect('send this file to your support representative.', 15) - self.assertEquals(grp, 0) - kill(sos.pid, SIGINT) - - -if __name__ == '__main__': - unittest.main() - -# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.2/tests/unittests/utilities_tests.py sosreport-4.3/tests/unittests/utilities_tests.py --- sosreport-4.2/tests/unittests/utilities_tests.py 2021-08-16 15:59:32.000000000 +0000 +++ sosreport-4.3/tests/unittests/utilities_tests.py 2022-02-15 04:20:20.000000000 +0000 @@ -78,9 +78,8 @@ def test_output_chdir(self): cmd = "/bin/bash -c 'echo $PWD'" result = sos_get_command_output(cmd, chdir=TEST_DIR) - print(result) self.assertEquals(result['status'], 0) - self.assertEquals(result['output'].strip(), TEST_DIR) + self.assertTrue(result['output'].strip().endswith(TEST_DIR)) def test_shell_out(self): self.assertEquals("executed\n", shell_out('echo executed')) diff -Nru sosreport-4.2/tests/vendor_tests/redhat/rhbz2018033.py sosreport-4.3/tests/vendor_tests/redhat/rhbz2018033.py --- sosreport-4.2/tests/vendor_tests/redhat/rhbz2018033.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.3/tests/vendor_tests/redhat/rhbz2018033.py 2022-02-15 04:20:20.000000000 +0000 @@ -0,0 +1,35 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos_tests import StageTwoReportTest + + +class rhbz2018033(StageTwoReportTest): + """Test that control of plugin timeouts is independent of other plugin + timeouts. See #2744. + + https://bugzilla.redhat.com/show_bug.cgi?id=2018033 + + :avocado: tags=stagetwo + """ + + install_plugins = ['timeout_test'] + sos_cmd = '-vvv -o timeout_test,networking -k timeout_test.timeout=1 --plugin-timeout=123' + + def test_timeouts_separate(self): + self.assertSosUILogContains('Plugin timeout_test timed out') + self.assertSosUILogNotContains('Plugin networking timed out') + + def test_timeout_manifest_recorded(self): + testm = self.get_plugin_manifest('timeout_test') + self.assertTrue(testm['timeout_hit']) + self.assertTrue(testm['timeout'] == 1) + + netm = self.get_plugin_manifest('networking') + self.assertFalse(netm['timeout_hit']) + self.assertTrue(netm['timeout'] == 123)