diff -Nru sosreport-4.5.4ubuntu0.23.04.1/.cirrus.yml sosreport-4.5.6/.cirrus.yml --- sosreport-4.5.4ubuntu0.23.04.1/.cirrus.yml 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/.cirrus.yml 2023-07-20 21:38:49.000000000 +0000 @@ -106,6 +106,7 @@ python3 setup.py sdist cp dist/sos*.tar.gz /rpmbuild/SOURCES/ rpmbuild -bs sos.spec + dnf -y builddep /rpmbuild/SRPMS/sos*src.rpm rpmbuild -bb sos.spec # Retrieving the built rpm in later tasks requires knowing the exact name # of the file. To avoid having to juggle version numbers here, rename it diff -Nru sosreport-4.5.4ubuntu0.23.04.1/debian/changelog sosreport-4.5.6/debian/changelog --- sosreport-4.5.4ubuntu0.23.04.1/debian/changelog 2023-06-13 06:17:32.000000000 +0000 +++ sosreport-4.5.6/debian/changelog 2023-07-24 04:30:32.000000000 +0000 @@ -1,13 +1,13 @@ -sosreport (4.5.4ubuntu0.23.04.1) lunar; urgency=medium +sosreport (4.5.6-0ubuntu1~23.04.1) lunar; urgency=medium - * New 4.5.4 upstream. (LP: #2022915) + * New 4.5.6 upstream. (LP: #2028327) * For more details, full release note is available here: - - https://github.com/sosreport/sos/releases/tag/4.5.4 + - https://github.com/sosreport/sos/releases/tag/4.5.6 * d/control: - - Add 'python3-pexpect' as part of the build depends. - - Add 'python3-magic' to runtime depends. + - Add 'python3-magic' as part of the runtime depends. + - Add 'python3-pexpect' as part of the build depends. * d/rules: - skip running unittests/policy_tests.py due to the avocado dependency @@ -15,10 +15,7 @@ * Remaining patches: - d/p/0001-debian-change-tmp-dir-location.patch - * New patches: - - d/p/0002-regex-flags.patch (LP: #2024547) - - -- Nikhil Kshirsagar Tue, 13 Jun 2023 06:17:32 +0000 + -- Nikhil Kshirsagar Mon, 24 Jul 2023 04:30:32 +0000 sosreport (4.5-1ubuntu0) lunar; urgency=medium diff -Nru sosreport-4.5.4ubuntu0.23.04.1/debian/patches/0002-regex-flags.patch sosreport-4.5.6/debian/patches/0002-regex-flags.patch --- sosreport-4.5.4ubuntu0.23.04.1/debian/patches/0002-regex-flags.patch 2023-06-13 06:14:19.000000000 +0000 +++ sosreport-4.5.6/debian/patches/0002-regex-flags.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,35 +0,0 @@ -Description: Fix do_file_private_sub() bug related to regex flags -Origin: upstream -Bug: https://github.com/sosreport/sos/issues/3261 -Applied-Upstream: https://github.com/sosreport/sos/pull/3263 ---- -This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ -Index: sos-4.5.4/sos/report/plugins/__init__.py -=================================================================== ---- sos-4.5.4.orig/sos/report/plugins/__init__.py -+++ sos-4.5.4/sos/report/plugins/__init__.py -@@ -1274,7 +1274,12 @@ class Plugin(): - """ - try: - path = self._get_dest_for_srcpath(srcpath) -- pattern = regexp.pattern if hasattr(regexp, "pattern") else regexp -+ if hasattr(regexp, "pattern"): -+ pattern = regexp.pattern -+ flags = regexp.flags | re.IGNORECASE -+ else: -+ pattern = regexp -+ flags = re.IGNORECASE - self._log_debug("substituting scrpath '%s'" % srcpath) - self._log_debug("substituting '%s' for '%s' in '%s'" - % (subst, pattern, path)) -@@ -1284,8 +1289,8 @@ class Plugin(): - content = readable.read() - if not isinstance(content, str): - content = content.decode('utf8', 'ignore') -- result, replacements = re.subn(regexp, subst, content, -- flags=re.IGNORECASE) -+ result, replacements = re.subn(pattern, subst, content, -+ flags=flags) - if replacements: - self.archive.add_string(result, srcpath) - else: diff -Nru sosreport-4.5.4ubuntu0.23.04.1/debian/patches/series sosreport-4.5.6/debian/patches/series --- sosreport-4.5.4ubuntu0.23.04.1/debian/patches/series 2023-06-13 06:12:42.000000000 +0000 +++ sosreport-4.5.6/debian/patches/series 2023-07-24 04:29:57.000000000 +0000 @@ -1,2 +1 @@ 0001-debian-change-tmp-dir-location.patch -0002-regex-flags.patch diff -Nru sosreport-4.5.4ubuntu0.23.04.1/docs/conf.py sosreport-4.5.6/docs/conf.py --- sosreport-4.5.4ubuntu0.23.04.1/docs/conf.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/docs/conf.py 2023-07-20 21:38:49.000000000 +0000 @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = '4.5.4' +version = '4.5.6' # The full version, including alpha/beta/rc tags. -release = '4.5.4' +release = '4.5.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/.github/workflows/snap.yaml sosreport-4.5.6/.github/workflows/snap.yaml --- sosreport-4.5.4ubuntu0.23.04.1/.github/workflows/snap.yaml 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/.github/workflows/snap.yaml 2023-07-20 21:38:49.000000000 +0000 @@ -3,10 +3,13 @@ push: branches: - main + release: + types: + - published jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 concurrency: group: snap-build cancel-in-progress: true @@ -25,8 +28,16 @@ - run: | sudo sos help - uses: snapcore/action-publish@v1 + if: ${{ github.event_name == 'push' }} env: SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }} with: snap: ${{ steps.build-snap.outputs.snap }} release: "latest/edge" + - uses: snapcore/action-publish@v1 + if: ${{ github.event_name == 'release' }} + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }} + with: + snap: ${{ steps.build-snap.outputs.snap }} + release: "latest/candidate" diff -Nru sosreport-4.5.4ubuntu0.23.04.1/man/en/sos.1 sosreport-4.5.6/man/en/sos.1 --- sosreport-4.5.4ubuntu0.23.04.1/man/en/sos.1 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/man/en/sos.1 2023-07-20 21:38:49.000000000 +0000 @@ -165,7 +165,7 @@ .BR sos.conf (5) .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-clean.1 sosreport-4.5.6/man/en/sos-clean.1 --- sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-clean.1 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/man/en/sos-clean.1 2023-07-20 21:38:49.000000000 +0000 @@ -135,7 +135,7 @@ .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-collect.1 sosreport-4.5.6/man/en/sos-collect.1 --- sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-collect.1 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/man/en/sos-collect.1 2023-07-20 21:38:49.000000000 +0000 @@ -395,4 +395,8 @@ .BR sos.conf (5) .SH MAINTAINER - Jake Hunsaker +.nf +Maintained on GitHub at https://github.com/sosreport/sos +.fi +.SH AUTHORS & CONTRIBUTORS +See \fBAUTHORS\fR file in the package documentation. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-collector.1 sosreport-4.5.6/man/en/sos-collector.1 --- sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-collector.1 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/man/en/sos-collector.1 2023-07-20 21:38:49.000000000 +0000 @@ -395,4 +395,8 @@ .BR sos.conf (5) .SH MAINTAINER - Jake Hunsaker +.nf +Maintained on GitHub at https://github.com/sosreport/sos +.fi +.SH AUTHORS & CONTRIBUTORS +See \fBAUTHORS\fR file in the package documentation. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-mask.1 sosreport-4.5.6/man/en/sos-mask.1 --- sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-mask.1 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/man/en/sos-mask.1 2023-07-20 21:38:49.000000000 +0000 @@ -135,7 +135,7 @@ .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-report.1 sosreport-4.5.6/man/en/sos-report.1 --- sosreport-4.5.4ubuntu0.23.04.1/man/en/sos-report.1 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/man/en/sos-report.1 2023-07-20 21:38:49.000000000 +0000 @@ -494,7 +494,7 @@ .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/man/en/sosreport.1 sosreport-4.5.6/man/en/sosreport.1 --- sosreport-4.5.4ubuntu0.23.04.1/man/en/sosreport.1 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/man/en/sosreport.1 2023-07-20 21:38:49.000000000 +0000 @@ -494,7 +494,7 @@ .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff -Nru sosreport-4.5.4ubuntu0.23.04.1/snap/snapcraft.yaml sosreport-4.5.6/snap/snapcraft.yaml --- sosreport-4.5.4ubuntu0.23.04.1/snap/snapcraft.yaml 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/snap/snapcraft.yaml 2023-07-20 21:38:49.000000000 +0000 @@ -19,13 +19,18 @@ override-pull: | craftctl default craftctl set version="$(git describe --tags --always)" + build-attributes: + - enable-patchelf build-packages: - git - python3 - snapcraft - gettext - stage-packages: - python3-venv + stage-packages: + - python3.10-minimal + - libpython3.10-minimal + - libpython3.10-stdlib python-packages: - pip - setuptools diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/archive.py sosreport-4.5.6/sos/archive.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/archive.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/archive.py 2023-07-20 21:38:49.000000000 +0000 @@ -73,7 +73,7 @@ # this is our contract to clients of the Archive class hierarchy. # All sub-classes need to implement these methods (or inherit concrete # implementations from a parent class. - def add_file(self, src, dest=None): + def add_file(self, src, dest=None, force=False): raise NotImplementedError def add_string(self, content, dest, mode='w'): @@ -344,12 +344,12 @@ self.log_debug("caught '%s' setting attributes of '%s'" % (e, dest)) - def add_file(self, src, dest=None): + def add_file(self, src, dest=None, force=False): with self._path_lock: if not dest: dest = src - dest = self.check_path(dest, P_FILE) + dest = self.check_path(dest, P_FILE, force=force) if not dest: return diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/archives/__init__.py sosreport-4.5.6/sos/cleaner/archives/__init__.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/archives/__init__.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/archives/__init__.py 2023-07-20 21:38:49.000000000 +0000 @@ -69,6 +69,14 @@ """Check if the archive is a well-known type we directly support""" return False + @property + def is_sos(self): + return 'sos' in self.__class__.__name__.lower() + + @property + def is_insights(self): + return 'insights' in self.type_name + def _load_self(self): if self.is_tarfile: self.tarobj = tarfile.open(self.archive_path) @@ -166,8 +174,12 @@ ) return '' else: - with open(self.format_file_name(fname), 'r') as to_read: - return to_read.read() + try: + with open(self.format_file_name(fname), 'r') as to_read: + return to_read.read() + except Exception as err: + self.log_debug(f"Failed to get contents of {fname}: {err}") + return '' def extract(self, quiet=False): if self.is_tarfile: diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/archives/insights.py sosreport-4.5.6/sos/cleaner/archives/insights.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/archives/insights.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/archives/insights.py 2023-07-20 21:38:49.000000000 +0000 @@ -22,12 +22,6 @@ type_name = 'insights' description = 'insights-client archive' - prep_files = { - 'hostname': 'data/insights_commands/hostname_-f', - 'ip': 'data/insights_commands/ip_addr', - 'mac': 'data/insights_commands/ip_addr' - } - @classmethod def check_is_type(cls, arc_path): try: diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/archives/sos.py sosreport-4.5.6/sos/cleaner/archives/sos.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/archives/sos.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/archives/sos.py 2023-07-20 21:38:49.000000000 +0000 @@ -22,24 +22,6 @@ type_name = 'report' description = 'sos report archive' - prep_files = { - 'hostname': [ - 'sos_commands/host/hostname', - 'etc/hosts' - ], - 'ip': 'sos_commands/networking/ip_-o_addr', - 'mac': 'sos_commands/networking/ip_-d_address', - 'username': [ - 'sos_commands/login/lastlog_-u_1000-60000', - 'sos_commands/login/lastlog_-u_60001-65536', - 'sos_commands/login/lastlog_-u_65537-4294967295', - # AD users will be reported here, but favor the lastlog files since - # those will include local users who have not logged in - 'sos_commands/login/last', - 'etc/cron.allow', - 'etc/cron.deny' - ] - } @classmethod def check_is_type(cls, arc_path): diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/__init__.py sosreport-4.5.6/sos/cleaner/__init__.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/__init__.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/__init__.py 2023-07-20 21:38:49.000000000 +0000 @@ -13,6 +13,7 @@ import logging import os import shutil +import sos.cleaner.preppers import tempfile from concurrent.futures import ThreadPoolExecutor @@ -31,7 +32,7 @@ SoSCollectorDirectory) from sos.cleaner.archives.generic import DataDirArchive, TarballArchive from sos.cleaner.archives.insights import InsightsArchive -from sos.utilities import get_human_readable +from sos.utilities import get_human_readable, import_module, ImporterHelper from textwrap import fill @@ -125,13 +126,12 @@ self.cleaner_md = self.manifest.components.add_section('cleaner') self.parsers = [ - SoSHostnameParser(self.cleaner_mapping, self.opts.domains), + SoSHostnameParser(self.cleaner_mapping), SoSIPParser(self.cleaner_mapping), SoSIPv6Parser(self.cleaner_mapping), SoSMacParser(self.cleaner_mapping), - SoSKeywordParser(self.cleaner_mapping, self.opts.keywords, - self.opts.keyword_file), - SoSUsernameParser(self.cleaner_mapping, self.opts.usernames) + SoSKeywordParser(self.cleaner_mapping), + SoSUsernameParser(self.cleaner_mapping) ] for _parser in self.opts.disable_parsers: @@ -363,6 +363,11 @@ # we have at least one valid target to obfuscate self.completed_reports = [] + # TODO: as we separate mappings and parsers further, do this in a less + # janky manner + for parser in self.parsers: + if parser.name == 'Hostname Parser': + parser.mapping.set_initial_counts() self.preload_all_archives_into_maps() self.generate_parser_item_regexes() self.obfuscate_report_paths() @@ -583,6 +588,63 @@ for parser in self.parsers: parser.generate_item_regexes() + def _prepare_archive_with_prepper(self, archive, prepper): + """ + For each archive we've determined we need to operate on, pass it to + each prepper so that we can extract necessary files and/or items for + direct regex replacement. Preppers define these methods per parser, + so it is possible that a single prepper will read the same file for + different parsers/mappings. This is preferable to the alternative of + building up monolithic lists of file paths, as we'd still need to + manipulate these on a per-archive basis. + + :param archive: The archive we are currently using to prepare our + mappings with + :type archive: ``SoSObfuscationArchive`` subclass + + :param prepper: The individual prepper we're using to source items + :type prepper: ``SoSPrepper`` subclass + """ + for _parser in self.parsers: + pname = _parser.name.lower().split()[0].strip() + for _file in prepper.get_parser_file_list(pname, archive): + content = archive.get_file_content(_file) + if not content: + continue + self.log_debug(f"Prepping {pname} parser with file {_file} " + f"from {archive.ui_name}") + for line in content.splitlines(): + try: + _parser.parse_line(line) + except Exception as err: + self.log_debug( + f"Failed to prep {pname} map from {_file}: {err}" + ) + map_items = prepper.get_items_for_map(pname, archive) + if map_items: + self.log_debug(f"Prepping {pname} mapping with items from " + f"{archive.ui_name}") + for item in map_items: + _parser.mapping.add(item) + + for ritem in prepper.regex_items[pname]: + _parser.mapping.add_regex_item(ritem) + + def get_preppers(self): + """ + Discover all locally available preppers so that we can prepare the + mappings with obfuscation matches in a controlled manner + + :returns: All preppers that can be leveraged locally + :rtype: A generator of `SoSPrepper` items + """ + helper = ImporterHelper(sos.cleaner.preppers) + preps = [] + for _prep in helper.get_modules(): + preps.extend(import_module(f"sos.cleaner.preppers.{_prep}")) + for prepper in sorted(preps, key=lambda x: x.priority): + yield prepper(options=self.opts) + def preload_all_archives_into_maps(self): """Before doing the actual obfuscation, if we have multiple archives to obfuscate then we need to preload each of them into the mappings @@ -590,42 +652,9 @@ obfuscated in node1's archive. """ self.log_info("Pre-loading all archives into obfuscation maps") - for _arc in self.report_paths: - for _parser in self.parsers: - try: - pfile = _arc.prep_files[_parser.name.lower().split()[0]] - if not pfile: - continue - except (IndexError, KeyError): - continue - if isinstance(pfile, str): - pfile = [pfile] - for parse_file in pfile: - self.log_debug("Attempting to load %s" % parse_file) - try: - content = _arc.get_file_content(parse_file) - if not content: - continue - if isinstance(_parser, SoSUsernameParser): - _parser.load_usernames_into_map(content) - elif isinstance(_parser, SoSHostnameParser): - if 'hostname' in parse_file: - _parser.load_hostname_into_map( - content.splitlines()[0] - ) - elif 'etc/hosts' in parse_file: - _parser.load_hostname_from_etc_hosts( - content - ) - else: - for line in content.splitlines(): - self.obfuscate_line(line) - except Exception as err: - self.log_info( - "Could not prepare %s from %s (archive: %s): %s" - % (_parser.name, parse_file, _arc.archive_name, - err) - ) + for prepper in self.get_preppers(): + for archive in self.report_paths: + self._prepare_archive_with_prepper(archive, prepper) def obfuscate_report(self, archive): """Individually handle each archive or directory we've discovered by @@ -749,7 +778,7 @@ % (short_name, err), caller=arc_name) tfile.seek(0) if subs: - shutil.copy(tfile.name, filename) + shutil.copyfile(tfile.name, filename) tfile.close() _ob_short_name = self.obfuscate_string(short_name.split('/')[-1]) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/mappings/hostname_map.py sosreport-4.5.6/sos/cleaner/mappings/hostname_map.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/mappings/hostname_map.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/mappings/hostname_map.py 2023-07-20 21:38:49.000000000 +0000 @@ -41,7 +41,7 @@ ] strip_exts = ('.yaml', '.yml', '.crt', '.key', '.pem', '.log', '.repo', - '.rules') + '.rules', '.conf', '.cfg') host_count = 0 domain_count = 0 @@ -80,10 +80,6 @@ self._domains[_domain_to_inject] = _ob_domain self.set_initial_counts() - def load_domains_from_options(self, domains): - for domain in domains: - self.sanitize_domain(domain.split('.')) - def get_regex_result(self, item): """Override the base get_regex_result() to provide a regex that, if this is an FQDN or a straight domain, will include an underscore diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/mappings/mac_map.py sosreport-4.5.6/sos/cleaner/mappings/mac_map.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/mappings/mac_map.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/mappings/mac_map.py 2023-07-20 21:38:49.000000000 +0000 @@ -75,5 +75,5 @@ if re.match('(([0-9a-fA-F]{4}:){3}([0-9a-fA-F]){4})', item): return self.mac6_quad_template % hextets # match 48-bit IPv4 MAC addresses - if re.match('([0-9a-fA-F]:?){12}', item): + if re.match('([0-9a-fA-F][:_]?){12}', item): return self.mac_template % hextets diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/mappings/username_map.py sosreport-4.5.6/sos/cleaner/mappings/username_map.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/mappings/username_map.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/mappings/username_map.py 2023-07-20 21:38:49.000000000 +0000 @@ -22,11 +22,6 @@ name_count = 0 - def load_names_from_options(self, opt_names): - for name in opt_names: - if name and name not in self.dataset.keys(): - self.add(name) - def sanitize_item(self, username): """Obfuscate a new username not currently found in the map """ diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/parsers/hostname_parser.py sosreport-4.5.6/sos/cleaner/parsers/hostname_parser.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/parsers/hostname_parser.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/parsers/hostname_parser.py 2023-07-20 21:38:49.000000000 +0000 @@ -21,14 +21,9 @@ r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}(\b|_)))' ] - def __init__(self, config, opt_domains=None): + def __init__(self, config): self.mapping = SoSHostnameMap() super(SoSHostnameParser, self).__init__(config) - self.mapping.load_domains_from_map() - self.mapping.load_domains_from_options(opt_domains) - self.short_names = [] - self.load_short_names_from_mapping() - self.mapping.set_initial_counts() def parse_line(self, line): """This will be called for every line in every file we process, so that @@ -47,55 +42,3 @@ line, _rcount = self._parse_line_with_compiled_regexes(line) count += _rcount return line, count - - def load_short_names_from_mapping(self): - """When we load the mapping file into the hostname map, we have to do - some dancing to get those loaded properly into the "intermediate" dicts - that the map uses to hold hosts and domains. Similarly, we need to also - extract shortnames known to the map here. - """ - for hname in self.mapping.dataset.keys(): - if len(hname.split('.')) == 1: - # we have a short name only with no domain - if hname not in self.short_names: - self.short_names.append(hname) - - def load_hostname_into_map(self, hostname_string): - """Force add the domainname found in /sos_commands/host/hostname into - the map. We have to do this here since the normal map prep approach - from the parser would be ignored since the system's hostname is not - guaranteed - """ - if 'localhost' in hostname_string: - return - domains = hostname_string.split('.') - if len(domains) > 1: - self.short_names.append(domains[0]) - else: - self.short_names.append(hostname_string) - if len(domains) > 3: - # make sure we implicitly get example.com if the system's hostname - # is something like foo.bar.example.com - high_domain = '.'.join(domains[-2:]) - self.mapping.add(high_domain) - self.mapping.add(hostname_string) - - def load_hostname_from_etc_hosts(self, content): - """Parse an archive's copy of /etc/hosts, which requires handling that - is separate from the output of the `hostname` command. Just like - load_hostname_into_map(), this has to be done explicitly and we - cannot rely upon the more generic methods to do this reliably. - """ - lines = content.splitlines() - for line in lines: - if line.startswith('#') or 'localhost' in line: - continue - hostln = line.split()[1:] - for host in hostln: - if len(host.split('.')) == 1: - # only generate a mapping for fqdns but still record the - # short name here for later obfuscation with parse_line() - self.short_names.append(host) - self.mapping.add_regex_item(host) - else: - self.mapping.add(host) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/parsers/keyword_parser.py sosreport-4.5.6/sos/cleaner/parsers/keyword_parser.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/parsers/keyword_parser.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/parsers/keyword_parser.py 2023-07-20 21:38:49.000000000 +0000 @@ -8,7 +8,6 @@ # # See the LICENSE file in the source distribution for further information. -import os from sos.cleaner.parsers import SoSCleanerParser from sos.cleaner.mappings.keyword_map import SoSKeywordMap @@ -21,23 +20,9 @@ name = 'Keyword Parser' map_file_key = 'keyword_map' - def __init__(self, config, keywords=None, keyword_file=None): + def __init__(self, config): self.mapping = SoSKeywordMap() - self.user_keywords = [] super(SoSKeywordParser, self).__init__(config) - for _keyword in self.mapping.dataset.keys(): - self.user_keywords.append(_keyword) - if keywords: - for keyword in keywords: - if keyword not in self.user_keywords: - # pre-generate an obfuscation mapping for each keyword - # this is necessary for cases where filenames are being - # obfuscated before or instead of file content - self.mapping.get(keyword.lower()) - self.user_keywords.append(keyword) - if keyword_file and os.path.exists(keyword_file): - with open(keyword_file, 'r') as kwf: - self.user_keywords.extend(kwf.read().splitlines()) def _parse_line(self, line): return line, 0 diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/parsers/mac_parser.py sosreport-4.5.6/sos/cleaner/parsers/mac_parser.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/parsers/mac_parser.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/parsers/mac_parser.py 2023-07-20 21:38:49.000000000 +0000 @@ -16,17 +16,17 @@ # aa:bb:cc:fe:ff:dd:ee:ff IPV6_REG_8HEX = ( r'((? + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.cleaner.preppers import SoSPrepper + + +class HostnamePrepper(SoSPrepper): + """ + Prepper for providing domain and hostname information to the hostname + mapping. + + The items from hostname sources are handled manually via the _get_items + method, rather than passing the file directly, as the parser does not know + what hostnames or domains to match on initially. + + This will also populate the regex_items list with local short names. + """ + + name = 'hostname' + + def _get_items_for_hostname(self, archive): + items = [] + _file = 'hostname' + if archive.is_sos: + _file = 'sos_commands/host/hostname' + elif archive.is_insights: + _file = 'data/insights_commands/hostname_-f' + + content = archive.get_file_content(_file) + if content and content != 'localhost': + domains = content.split('.') + if len(domains) > 1: + items.append(domains[0]) + self.regex_items['hostname'].add((domains[0])) + if len(domains) > 3: + # make sure we get example.com if the system's hostname + # is something like foo.bar.example.com + top_domain = '.'.join(domains[-2:]) + items.append(top_domain.strip()) + items.append(content.strip()) + + _hosts = archive.get_file_content('etc/hosts') + for line in _hosts.splitlines(): + if line.startswith('#') or 'localhost' in line: + continue + hostln = line.split()[1:] + for host in hostln: + if len(host.split('.')) == 1: + self.regex_items['hostname'].add(host) + else: + items.append(host) + + for domain in self.opts.domains: + items.append(domain) + + return items diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/__init__.py sosreport-4.5.6/sos/cleaner/preppers/__init__.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/preppers/__init__.py 2023-07-20 21:38:49.000000000 +0000 @@ -0,0 +1,126 @@ +# Copyright 2023 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import logging + + +class SoSPrepper(): + """ + A prepper is a way to prepare loaded mappings with selected items within + an sos report prior to beginning the full obfuscation routine. + + This was previously handled directly within archives, however this is a bit + cumbersome and doesn't allow for all the flexibility we could use in this + effort. + + Preppers are separated from parsers but will leverage them in order to feed + parser-matched strings from files highlighted by a Prepper() to the + appropriate mapping for initial obfuscation. + + Preppers may specify their own priority in order to influence the order in + which mappings are prepped. Further, Preppers have two ways to prepare + the maps - either by generating a list of filenames or via directly pulling + content out of select files without the assistance of a parser. A lower + priority value means the prepper should run sooner than those with higher + values. + + For the former approach, `Prepper._get_$parser_file_list()` should be used + and should yield filenames that exist in target archives. For the latter, + the `Prepper._get_items_for_$map()` should be used. + + Finally, a `regex_items` dict is available for storing individual regex + items for parsers that rely on them. These items will be added after all + files and other individual items are handled. This dict has keys set to + parser/mapping names, and the values should be sets of items, so preppers + should add to them like so: + + self.regex_items['hostname'].add('myhostname') + """ + + name = 'Undefined' + priority = 100 + + def __init__(self, options): + self.regex_items = { + 'hostname': set(), + 'ip': set(), + 'ipv6': set(), + 'keyword': set(), + 'mac': set(), + 'username': set() + } + self.opts = options + self.soslog = logging.getLogger('sos') + self.ui_log = logging.getLogger('sos_ui') + + def _fmt_log_msg(self, msg): + return f"[prepper:{self.name}] {msg}" + + def log_debug(self, msg): + self.soslog.debug(self._fmt_log_msg(msg)) + + def log_info(self, msg): + self.soslog.info(self._fmt_log_msg(msg)) + + def log_error(self, msg): + self.soslog.error(self._fmt_log_msg(msg)) + + def get_parser_file_list(self, parser, archive): + """ + Helper that calls the appropriate Prepper method for the specified + parser. This allows Preppers to be able to provide items for multiple + types of parsers without needing to handle repetitious logic to + determine which parser we're interested within each individual call. + + The convention to use is to define `_get_$parser_file_list()` methods + within Preppers, e.g. `_get_hostname_file_list()` would be used to + provide filenames for the hostname parser. If such a method is not + defined within a Prepper for a given parser, we handle that here so + that individual Preppers do not need to. + + :param parser: The _name_ of the parser to get a file list for + :type parser: ``str`` + + :param archive: The archive we are operating on currently for the + specified parser + :type archive: ``SoSObfuscationArchive`` + + :returns: A list of filenames within the archive to prep with + :rtype: ``list`` + """ + _check = f"_get_{parser}_file_list" + if hasattr(self, _check): + return getattr(self, _check)(archive) + return [] + + def get_items_for_map(self, mapping, archive): + """ + Similar to `get_parser_file_list()`, a helper for calling the specific + method for generating items for the given `map`. This allows Preppers + to be able to provide items for multiple types of maps, without the + need to handle repetitious logic to determine which parser we're + interested in within each individual call. + + :param mapping: The _name_ of the mapping to get items for + :type mapping: ``str`` + + :param archive: The archive we are operating on currently for the + specified parser + :type archive: ``SoSObfuscationArchive`` + + :returns: A list of distinct items to obfuscate without using a parser + :rtype: ``list`` + """ + _check = f"_get_items_for_{mapping}" + if hasattr(self, _check): + return getattr(self, _check)(archive) + return [] + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/ip.py sosreport-4.5.6/sos/cleaner/preppers/ip.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/ip.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/preppers/ip.py 2023-07-20 21:38:49.000000000 +0000 @@ -0,0 +1,34 @@ +# Copyright 2023 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.cleaner.preppers import SoSPrepper + + +class IPPrepper(SoSPrepper): + """ + This prepper is for IP network addresses. The aim of this prepper is to + provide the file path for where the output of `ip addr` is saved. + """ + + name = 'ip' + + def _get_ipv6_file_list(self, archive): + return self._get_ip_file_list(archive) + + def _get_ip_file_list(self, archive): + _files = [] + if archive.is_sos: + _files = ['sos_commands/networking/ip_-o_addr'] + elif archive.is_insights: + _files = ['data/insights_commands/ip_addr'] + + return _files + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/keywords.py sosreport-4.5.6/sos/cleaner/preppers/keywords.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/keywords.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/preppers/keywords.py 2023-07-20 21:38:49.000000000 +0000 @@ -0,0 +1,37 @@ +# Copyright 2023 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os + +from sos.cleaner.preppers import SoSPrepper + + +class KeywordPrepper(SoSPrepper): + """ + Prepper to handle keywords passed to cleaner via either the `--keywords` + or `--keyword-file` options. + """ + + name = 'keyword' + + def _get_items_for_keyword(self, archive): + items = [] + for kw in self.opts.keywords: + items.append(kw) + if self.opts.keyword_file and os.path.exists(self.opts.keyword_file): + with open(self.opts.keyword_file, 'r') as kwf: + items.extend(kwf.read().splitlines()) + + for item in items: + self.regex_items['keyword'].add(item) + + return items + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/mac.py sosreport-4.5.6/sos/cleaner/preppers/mac.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/mac.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/preppers/mac.py 2023-07-20 21:38:49.000000000 +0000 @@ -0,0 +1,28 @@ +# Copyright 2023 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.cleaner.preppers import SoSPrepper + + +class MacPrepper(SoSPrepper): + """ + Prepper for sourcing the host's MAC address in order to prep the mapping. + """ + + name = 'mac' + + def _get_mac_file_list(self, archive): + if archive.is_sos: + return ['sos_commands/networking/ip_-d_address'] + elif archive.is_insights: + return ['data/insights_commands/ip_addr'] + return [] + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/usernames.py sosreport-4.5.6/sos/cleaner/preppers/usernames.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/cleaner/preppers/usernames.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.5.6/sos/cleaner/preppers/usernames.py 2023-07-20 21:38:49.000000000 +0000 @@ -0,0 +1,69 @@ +# Copyright 2023 Red Hat, Inc. Jake Hunsaker + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.cleaner.preppers import SoSPrepper + + +class UsernamePrepper(SoSPrepper): + """ + This prepper is used to source usernames from various `last` output content + as well as a couple select files. This prepper will also leverage the + --usernames option. + """ + + name = 'username' + + skip_list = [ + 'core', + 'nobody', + 'nfsnobody', + 'shutdown', + 'stack', + 'reboot', + 'root', + 'ubuntu', + 'username', + 'wtmp' + ] + + def _get_items_for_username(self, archive): + items = set() + _files = [ + 'sos_commands/login/lastlog_-u_1000-60000', + 'sos_commands/login/lastlog_-u_60001-65536', + 'sos_commands/login/lastlog_-u_65537-4294967295', + # AD users will be reported here, but favor the lastlog files since + # those will include local users who have not logged in + 'sos_commands/login/last', + 'etc/cron.allow', + 'etc/cron.deny' + ] + for _file in _files: + content = archive.get_file_content(_file) + if not content: + continue + for line in content.splitlines(): + try: + user = line.split()[0].lower() + if user and user not in self.skip_list: + items.add(user) + if '\\' in user: + items.add(user.split('\\')[-1]) + except Exception: + # empty line or otherwise unusable for name sourcing + pass + + for opt_user in self.opts.usernames: + if opt_user not in self.skip_list: + items.add(opt_user) + + return items + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/collector/clusters/__init__.py sosreport-4.5.6/sos/collector/clusters/__init__.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/collector/clusters/__init__.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/collector/clusters/__init__.py 2023-07-20 21:38:49.000000000 +0000 @@ -294,7 +294,7 @@ """ return node.address == self.primary.address - def exec_primary_cmd(self, cmd, need_root=False): + def exec_primary_cmd(self, cmd, need_root=False, timeout=180): """Used to retrieve command output from a (primary) node in a cluster :param cmd: The command to run @@ -303,11 +303,15 @@ :param need_root: Does the command require root privileges :type need_root: ``bool`` + :param timeout: Amount of time to allow cmd to run in seconds + :type timeout: ``int`` + :returns: The output and status of `cmd` :rtype: ``dict`` """ pty = self.primary.local is False - res = self.primary.run_command(cmd, get_pty=pty, need_root=need_root) + res = self.primary.run_command(cmd, get_pty=pty, need_root=need_root, + timeout=timeout) if res['output']: res['output'] = res['output'].replace('Password:', '') return res diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/collector/clusters/ocp.py sosreport-4.5.6/sos/collector/clusters/ocp.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/collector/clusters/ocp.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/collector/clusters/ocp.py 2023-07-20 21:38:49.000000000 +0000 @@ -86,7 +86,7 @@ _oc_path['output'].strip().lstrip('/') ) else: - self.log_warning( + self.log_warn( "Unable to to determine PATH for 'oc' command, " "node enumeration may fail." ) @@ -172,19 +172,32 @@ """Remove the project we created to execute within """ if self.project: - ret = self.exec_primary_cmd( - self.fmt_oc_cmd("delete project %s" % self.project) - ) - if not ret['status'] == 0: - self.log_error("Error deleting temporary project: %s" - % ret['output']) - ret = self.exec_primary_cmd( - self.fmt_oc_cmd("wait namespace/%s --for=delete --timeout=30s" - % self.project) - ) - if not ret['status'] == 0: - self.log_error("Error waiting for temporary project to be " - "deleted: %s" % ret['output']) + try: + ret = self.exec_primary_cmd( + self.fmt_oc_cmd(f"delete project {self.project}"), + timeout=30 + ) + if not ret['status'] == 0: + self.log_error( + f"Error deleting temporary project: {ret['output']}" + ) + ret = self.exec_primary_cmd( + self.fmt_oc_cmd( + f"wait namespace/{self.project} --for=delete " + f"--timeout=30s" + ) + ) + if not ret['status'] == 0: + self.log_error( + f"Error waiting for temporary project to be deleted: " + f"{ret['output']}" + ) + except Exception as err: + self.log_error( + f"Failed attempting to remove temporary project " + f"'sos-collect-tmp': {err}\n" + f"Please manually remove the temporary project." + ) # don't leave the config on a non-existing project self.exec_primary_cmd(self.fmt_oc_cmd("project default")) self.project = None diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/collector/sosnode.py sosreport-4.5.6/sos/collector/sosnode.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/collector/sosnode.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/collector/sosnode.py 2023-07-20 21:38:49.000000000 +0000 @@ -808,8 +808,9 @@ except CommandTimeoutException: self.log_error('Timeout exceeded') raise - except Exception as e: - self.log_error('Error running sos report: %s' % e) + except Exception as err: + self.log_info(f"Exception during sos report execution: {err}") + self.ui_msg(f"Error running sos report: {err}") raise def retrieve_file(self, path): diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/__init__.py sosreport-4.5.6/sos/__init__.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/__init__.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/__init__.py 2023-07-20 21:38:49.000000000 +0000 @@ -14,7 +14,7 @@ This module houses the i18n setup and message function. The default is to use gettext to internationalize messages. """ -__version__ = "4.5.4" +__version__ = "4.5.6" import os import sys diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/options.py sosreport-4.5.6/sos/options.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/options.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/options.py 2023-07-20 21:38:49.000000000 +0000 @@ -226,7 +226,7 @@ try: try: with open(config_file) as f: - config.readfp(f) + config.read_file(f, config_file) except DuplicateOptionError as err: raise exit("Duplicate option '%s' in section '%s' in file %s" % (err.option, err.section, config_file)) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/apache.py sosreport-4.5.6/sos/report/plugins/apache.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/apache.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/apache.py 2023-07-20 21:38:49.000000000 +0000 @@ -99,7 +99,7 @@ # relevant config files within each etcdirs = ["/etc/httpd%s" % ver for ver in vers] confs = [ - "conf/httpd.conf", + "conf/*.conf", "conf.d/*.conf", "conf.modules.d/*.conf" ] diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_ansible.py sosreport-4.5.6/sos/report/plugins/ceph_ansible.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_ansible.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_ansible.py 2023-07-20 21:38:49.000000000 +0000 @@ -14,7 +14,7 @@ short_desc = 'CEPH distributed storage - Ansible installer' plugin_name = 'ceph_ansible' - profiles = ('storage',) + profiles = ('storage', 'ceph') packages = ('ceph-ansible',) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_common.py sosreport-4.5.6/sos/report/plugins/ceph_common.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_common.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_common.py 2023-07-20 21:38:49.000000000 +0000 @@ -15,7 +15,7 @@ short_desc = 'CEPH common' plugin_name = 'ceph_common' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?(mon|rgw|osd).*',) ceph_hostname = gethostname() @@ -41,45 +41,53 @@ # This check will enable the plugin regardless of being # containerized or not - files = ('/etc/ceph/ceph.conf',) + files = ('/etc/ceph/ceph.conf', + '/var/snap/microceph/*',) def setup(self): all_logs = self.get_option("all_logs") - self.add_file_tags({ - '.*/ceph.conf': 'ceph_conf', - '/var/log/ceph(.*)?/ceph.log.*': 'ceph_log', - }) - - if not all_logs: - self.add_copy_spec("/var/log/calamari/*.log",) + microceph_pkg = self.policy.package_manager.pkg_by_name('microceph') + if not microceph_pkg: + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + '/var/log/ceph(.*)?/ceph.log.*': 'ceph_log', + }) + + if not all_logs: + self.add_copy_spec("/var/log/calamari/*.log",) + else: + self.add_copy_spec("/var/log/calamari",) + + self.add_copy_spec([ + "/var/log/ceph/**/ceph.log", + "/var/log/ceph/**/ceph.audit.log*", + "/var/log/calamari/*.log", + "/etc/ceph/", + "/etc/calamari/", + "/var/lib/ceph/tmp/", + ]) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) else: - self.add_copy_spec("/var/log/calamari",) - - self.add_copy_spec([ - "/var/log/ceph/**/ceph.log", - "/var/log/ceph/**/ceph.audit.log*", - "/var/log/calamari/*.log", - "/etc/ceph/", - "/etc/calamari/", - "/var/lib/ceph/tmp/", - ]) + self.add_copy_spec([ + "/var/snap/microceph/common/logs/ceph.log", + "/var/snap/microceph/common/logs/ceph.audit.log", + ]) self.add_cmd_output([ "ceph -v", ]) - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/*keyring*", - "/var/lib/ceph/*/*keyring*", - "/var/lib/ceph/*/*/*keyring*", - "/var/lib/ceph/osd", - "/var/lib/ceph/mon", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) - # vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_iscsi.py sosreport-4.5.6/sos/report/plugins/ceph_iscsi.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_iscsi.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_iscsi.py 2023-07-20 21:38:49.000000000 +0000 @@ -14,7 +14,7 @@ short_desc = "CEPH iSCSI" plugin_name = "ceph_iscsi" - profiles = ("storage", "virt", "container") + profiles = ("storage", "virt", "container", "ceph") packages = ("ceph-iscsi",) services = ("rbd-target-api", "rbd-target-gw") containers = ("rbd-target-api.*", "rbd-target-gw.*") diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_mds.py sosreport-4.5.6/sos/report/plugins/ceph_mds.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_mds.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_mds.py 2023-07-20 21:38:49.000000000 +0000 @@ -12,9 +12,9 @@ class CephMDS(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = 'CEPH mds' plugin_name = 'ceph_mds' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?fs.*',) - files = ('/var/lib/ceph/mds/',) + files = ('/var/lib/ceph/mds/*',) def setup(self): self.add_file_tags({ diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_mgr.py sosreport-4.5.6/sos/report/plugins/ceph_mgr.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_mgr.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_mgr.py 2023-07-20 21:38:49.000000000 +0000 @@ -35,8 +35,8 @@ short_desc = 'CEPH mgr' plugin_name = 'ceph_mgr' - profiles = ('storage', 'virt', 'container') - files = ('/var/lib/ceph/mgr/', '/var/lib/ceph/*/mgr*') + profiles = ('storage', 'virt', 'container', 'ceph') + files = ('/var/lib/ceph/mgr/*', '/var/lib/ceph/*/mgr*') containers = ('ceph-(.*-)?mgr.*',) def setup(self): diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_mon.py sosreport-4.5.6/sos/report/plugins/ceph_mon.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_mon.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_mon.py 2023-07-20 21:38:49.000000000 +0000 @@ -14,8 +14,8 @@ class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): """ This plugin serves to collect information on monitor nodes within a Ceph - cluster. It is designed to collect from several versions of Ceph, including - those versions that serve as the basis for RHCS 4 and RHCS 5. + or microceph cluster. It is designed to collect from several versions of + Ceph, including versions that serve as the basis for RHCS 4 and RHCS 5. Older versions of Ceph will have collections from locations such as /var/log/ceph, whereas newer versions (as of this plugin's latest update) @@ -32,37 +32,54 @@ short_desc = 'CEPH mon' plugin_name = 'ceph_mon' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') # note: for RHCS 5 / Ceph v16 the containers serve as an enablement trigger # but by default they are not capable of running various ceph commands in # this plugin - the `ceph` binary is functional directly on the host containers = ('ceph-(.*-)?mon.*',) - files = ('/var/lib/ceph/mon/', '/var/lib/ceph/*/mon*') + files = ('/var/lib/ceph/mon/*', '/var/lib/ceph/*/mon*', + '/var/snap/microceph/common/data/mon/*') ceph_version = 0 def setup(self): self.ceph_version = self.get_ceph_version() - self.add_file_tags({ - '.*/ceph.conf': 'ceph_conf', - "/var/log/ceph/(.*/)?ceph-.*mon.*.log": 'ceph_mon_log' - }) - - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/**/*keyring*", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/**/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) - - self.add_copy_spec([ - "/run/ceph/**/ceph-mon*", - "/var/lib/ceph/**/kv_backend", - "/var/log/ceph/**/*ceph-mon*.log" - ]) + microceph_pkg = self.policy.package_manager.pkg_by_name('microceph') + if not microceph_pkg: + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + "/var/log/ceph/(.*/)?ceph-.*mon.*.log": 'ceph_mon_log' + }) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/**/*keyring*", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/**/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + self.add_copy_spec([ + "/run/ceph/**/ceph-mon*", + "/var/lib/ceph/**/kv_backend", + "/var/log/ceph/**/*ceph-mon*.log" + ]) + + else: + self.add_forbidden_path([ + "/var/snap/microceph/common/**/*keyring*", + "/var/snap/microceph/current/**/*keyring*", + "/var/snap/microceph/common/data/mon/*/store.db", + "/var/snap/microceph/common/state/*", + ]) + + self.add_copy_spec([ + "/var/snap/microceph/common/data/mon/*", + "/var/snap/microceph/common/logs/*ceph-mon*.log", + "/var/snap/microceph/current/conf/*", + ]) self.add_cmd_output("ceph report", tags="ceph_report") self.add_cmd_output([ diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_osd.py sosreport-4.5.6/sos/report/plugins/ceph_osd.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_osd.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_osd.py 2023-07-20 21:38:49.000000000 +0000 @@ -29,64 +29,79 @@ short_desc = 'CEPH osd' plugin_name = 'ceph_osd' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?osd.*',) - files = ('/var/lib/ceph/osd/', '/var/lib/ceph/*/osd*') + files = ('/var/lib/ceph/osd/*', '/var/lib/ceph/*/osd*', + '/var/snap/microceph/common/data/osd/*') def setup(self): - - self.add_file_tags({ - "/var/log/ceph/(.*/)?ceph-(.*-)?osd.*.log": 'ceph_osd_log', - }) - - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/**/*keyring*", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/**/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) - - # Only collect OSD specific files - self.add_copy_spec([ - "/run/ceph/**/ceph-osd*", - "/var/lib/ceph/**/kv_backend", - "/var/log/ceph/**/ceph-osd*.log", - "/var/log/ceph/**/ceph-volume*.log", - ]) - - self.add_cmd_output([ - "ceph-disk list", - "ceph-volume lvm list" - ]) - - cmds = [ - "bluestore bluefs available", - "config diff", - "config show", - "dump_blacklist", - "dump_blocked_ops", - "dump_historic_ops_by_duration", - "dump_historic_slow_ops", - "dump_mempools", - "dump_ops_in_flight", - "dump_op_pq_state", - "dump_osd_network", - "dump_reservations", - "dump_watchers", - "log dump", - "perf dump", - "perf histogram dump", - "objecter_requests", - "ops", - "status", - "version", - ] - - self.add_cmd_output( - [f"ceph daemon {i} {c}" for i in self.get_socks() for c in cmds] - ) + microceph_pkg = self.policy.package_manager.pkg_by_name('microceph') + if not microceph_pkg: + self.add_file_tags({ + "/var/log/ceph/(.*/)?ceph-(.*-)?osd.*.log": 'ceph_osd_log', + }) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/**/*keyring*", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/**/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + # Only collect OSD specific files + self.add_copy_spec([ + "/run/ceph/**/ceph-osd*", + "/var/lib/ceph/**/kv_backend", + "/var/log/ceph/**/ceph-osd*.log", + "/var/log/ceph/**/ceph-volume*.log", + ]) + + self.add_cmd_output([ + "ceph-disk list", + "ceph-volume lvm list" + ]) + + cmds = [ + "bluestore bluefs available", + "config diff", + "config show", + "dump_blacklist", + "dump_blocked_ops", + "dump_historic_ops_by_duration", + "dump_historic_slow_ops", + "dump_mempools", + "dump_ops_in_flight", + "dump_op_pq_state", + "dump_osd_network", + "dump_reservations", + "dump_watchers", + "log dump", + "perf dump", + "perf histogram dump", + "objecter_requests", + "ops", + "status", + "version", + ] + + self.add_cmd_output( + [f"ceph daemon {i} {c}" for i in self.get_socks() for c in cmds] + ) + + else: + # Only collect microceph files, don't run any commands + self.add_forbidden_path([ + "/var/snap/microceph/common/**/*keyring*", + "/var/snap/microceph/current/**/*keyring*", + "/var/snap/microceph/common/state/*", + ]) + + self.add_copy_spec([ + "/var/snap/microceph/common/data/osd/*", + "/var/snap/microceph/common/logs/*ceph-osd*.log", + ]) def get_socks(self): """ diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_rgw.py sosreport-4.5.6/sos/report/plugins/ceph_rgw.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ceph_rgw.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ceph_rgw.py 2023-07-20 21:38:49.000000000 +0000 @@ -14,9 +14,9 @@ short_desc = 'CEPH rgw' plugin_name = 'ceph_rgw' - profiles = ('storage', 'virt', 'container', 'webserver') + profiles = ('storage', 'virt', 'container', 'webserver', 'ceph') containers = ('ceph-(.*)?rgw.*',) - files = ('/var/lib/ceph/radosgw',) + files = ('/var/lib/ceph/radosgw/*',) def setup(self): self.add_copy_spec('/var/log/ceph/ceph-client.rgw*.log', diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/collectd.py sosreport-4.5.6/sos/report/plugins/collectd.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/collectd.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/collectd.py 2023-07-20 21:38:49.000000000 +0000 @@ -51,7 +51,7 @@ "Password", "User", "[<]*URL", "Address" ] - regexp = r"((?m)^[#]*\s*(%s)\s* \s*)(.*)" % "|".join(protect_keys) + regexp = r"(^[#]*\s*(%s)\s* \s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub( "/etc/collectd.d/*.conf", regexp, r'\1"*********"' diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/conntrack.py sosreport-4.5.6/sos/report/plugins/conntrack.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/conntrack.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/conntrack.py 2023-07-20 21:38:49.000000000 +0000 @@ -7,7 +7,8 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt +from sos.report.plugins import (Plugin, IndependentPlugin, SoSPredicate, + PluginOpt) class Conntrack(Plugin, IndependentPlugin): @@ -37,11 +38,18 @@ "conntrackd -s expect", ]) - # Collect info from conntrack + # Collect info from conntrack, only when all required kmods are loaded + ct_pred = SoSPredicate(self, + kmods=['nf_conntrack', + 'nf_conntrack_netlink', + 'nf_defrag_ipv4', + 'nf_defrag_ipv6', + 'nfnetlink'], + required={'kmods': 'all'}) self.add_cmd_output([ "conntrack -L -o extended", "conntrack -S", - ]) + ], pred=ct_pred) # Capture additional data from namespaces; each command is run # per-namespace diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/cxl.py sosreport-4.5.6/sos/report/plugins/cxl.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/cxl.py 1970-01-01 00:00:00.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/cxl.py 2023-07-20 21:38:49.000000000 +0000 @@ -0,0 +1,46 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, IndependentPlugin + + +class cxl(Plugin, IndependentPlugin): + """This plugin collects data from Compute Express Link (CXL) devices + """ + + short_desc = 'Compute Express Link (CXL)' + plugin_name = 'cxl' + profiles = ('storage', 'hardware', 'memory') + # Utilities can be installed by package or self compiled + packages = ('cxl-cli', 'daxctl') + commands = ('cxl', 'daxctl') + + def setup(self): + """ Use the daxctl-list(1) command to collect disabled, devices, + mapping, and region information + + Output is JSON formatted + """ + self.add_cmd_output([ + "daxctl version", + "daxctl list", + "daxctl list -iDRM" + ]) + + """ Use the cxl-list(1) command to collect data about + all CXL devices. + + Output is JSON formatted + """ + self.add_cmd_output([ + "cxl version", + "cxl list", + "cxl list -vvv" + ]) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/docker.py sosreport-4.5.6/sos/report/plugins/docker.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/docker.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/docker.py 2023-07-20 21:38:49.000000000 +0000 @@ -9,7 +9,8 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import (Plugin, RedHatPlugin, UbuntuPlugin, - SoSPredicate, CosPlugin, PluginOpt) + SoSPredicate, CosPlugin, PluginOpt, + DebianPlugin) class Docker(Plugin, CosPlugin): @@ -133,7 +134,7 @@ ]) -class UbuntuDocker(Docker, UbuntuPlugin): +class UbuntuDocker(Docker, UbuntuPlugin, DebianPlugin): packages = ('docker.io', 'docker-engine', 'docker-ce', 'docker-ee') diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/grafana.py sosreport-4.5.6/sos/report/plugins/grafana.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/grafana.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/grafana.py 2023-07-20 21:38:49.000000000 +0000 @@ -42,6 +42,6 @@ "admin_password", "secret_key" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub("/etc/grafana/grafana.ini", regexp, r"\1*********") diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/grub2.py sosreport-4.5.6/sos/report/plugins/grub2.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/grub2.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/grub2.py 2023-07-20 21:38:49.000000000 +0000 @@ -31,9 +31,12 @@ "/boot/grub2/grubenv", "/boot/grub/grub.cfg", "/boot/loader/entries", + "/boot/grub2/custom.cfg", + "/boot/grub2/user.cfg", "/etc/default/grub", "/etc/grub2.cfg", - "/etc/grub.d" + "/etc/grub.d", + "/etc/grub2-efi.cfg" ]) self.add_cmd_output("ls -lanR /boot", tags="ls_boot") diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/__init__.py sosreport-4.5.6/sos/report/plugins/__init__.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/__init__.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/__init__.py 2023-07-20 21:38:49.000000000 +0000 @@ -1274,7 +1274,13 @@ """ try: path = self._get_dest_for_srcpath(srcpath) - pattern = regexp.pattern if hasattr(regexp, "pattern") else regexp + common_flags = re.IGNORECASE | re.MULTILINE + if hasattr(regexp, "pattern"): + pattern = regexp.pattern + flags = regexp.flags | common_flags + else: + pattern = regexp + flags = common_flags self._log_debug("substituting scrpath '%s'" % srcpath) self._log_debug("substituting '%s' for '%s' in '%s'" % (subst, pattern, path)) @@ -1284,10 +1290,10 @@ content = readable.read() if not isinstance(content, str): content = content.decode('utf8', 'ignore') - result, replacements = re.subn(regexp, subst, content, - flags=re.IGNORECASE) + result, replacements = re.subn(pattern, subst, content, + flags=flags) if replacements: - self.archive.add_string(result, srcpath) + self.archive.add_string(result, self.strip_sysroot(srcpath)) else: replacements = 0 except (OSError, IOError) as e: @@ -1380,7 +1386,11 @@ # skip recursive copying of symlink pointing to itself. if (absdest != srcpath): - self._do_copy_path(absdest) + # this allows for ensuring we collect the host's file when copying + # a symlink from within a container that is within the set sysroot + force = (absdest.startswith(self.sysroot) and + self.policy._in_container) + self._do_copy_path(absdest, force=force) else: self._log_debug("link '%s' points to itself, skipping target..." % linkdest) @@ -1442,7 +1452,7 @@ self.archive.add_node(path, mode, os.makedev(dev_maj, dev_min)) # Methods for copying files and shelling out - def _do_copy_path(self, srcpath, dest=None): + def _do_copy_path(self, srcpath, dest=None, force=False): """Copy file or directory to the destination tree. If a directory, then everything below it is recursively copied. A list of copied files are saved for use later in preparing a report. @@ -1494,7 +1504,7 @@ # FIXME: reflect permissions in archive self.archive.add_string("", dest) else: - self.archive.add_file(srcpath, dest) + self.archive.add_file(srcpath, dest, force=force) self.copied_files.append({ 'srcpath': srcpath, @@ -1502,6 +1512,8 @@ 'symlink': "no" }) + return + def add_forbidden_path(self, forbidden): """Specify a path, or list of paths, to not copy, even if it's part of an ``add_copy_spec()`` call @@ -2165,7 +2177,7 @@ if name: cmd_output_path = os.path.join(cmd_output_path, name) if make: - os.makedirs(cmd_output_path) + os.makedirs(cmd_output_path, exist_ok=True) return cmd_output_path diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ipa.py sosreport-4.5.6/sos/report/plugins/ipa.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ipa.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ipa.py 2023-07-20 21:38:49.000000000 +0000 @@ -88,6 +88,7 @@ self.add_copy_spec([ "/var/log/ipaserver-install.log", "/var/log/ipaserver-kra-install.log", + "/var/log/ipaserver-enable-sid.log", "/var/log/ipareplica-install.log", "/var/log/ipareplica-ca-install.log", "/var/log/ipa-custodia.audit.log" @@ -120,7 +121,8 @@ "/var/lib/ipa/certs/httpd.crt", "/var/kerberos/krb5kdc/kdc.crt", "/var/lib/ipa/sysrestore/sysrestore.state", - "/var/log/ipa/healthcheck/healthcheck.log*" + "/var/log/ipa/healthcheck/healthcheck.log*", + "/var/log/ipaepn.log*" ]) # Make sure to use the right PKI config and NSS DB folders diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/juju.py sosreport-4.5.6/sos/report/plugins/juju.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/juju.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/juju.py 2023-07-20 21:38:49.000000000 +0000 @@ -63,7 +63,7 @@ ] # Redact simple yaml style "key: value". - keys_regex = r"((?m)^\s*(%s)\s*:\s*)(.*)" % "|".join(protect_keys) + keys_regex = r"(^\s*(%s)\s*:\s*)(.*)" % "|".join(protect_keys) sub_regex = r"\1*********" self.do_path_regex_sub(agents_path, keys_regex, sub_regex) # Redact certificates diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/kernel.py sosreport-4.5.6/sos/report/plugins/kernel.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/kernel.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/kernel.py 2023-07-20 21:38:49.000000000 +0000 @@ -7,6 +7,7 @@ # See the LICENSE file in the source distribution for further information. from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt +from sos.policies.distros.redhat import RedHatPolicy import glob @@ -42,6 +43,11 @@ ] def setup(self): + # RedHat distributions can deliver kernel in RPM named either 'kernel' + # or 'kernel-redhat', so we must verify both + if isinstance(self.policy, RedHatPolicy): + self.verify_packages = ('kernel$', 'kernel-redhat$') + # compat self.add_cmd_output("uname -a", root_symlink="uname", tags="uname") self.add_cmd_output("lsmod", root_symlink="lsmod", tags="lsmod") diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/lustre.py sosreport-4.5.6/sos/report/plugins/lustre.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/lustre.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/lustre.py 2023-07-20 21:38:49.000000000 +0000 @@ -35,25 +35,44 @@ "lnetctl net show -v" ]) - self.get_params("basic", ["version", "health_check", "debug"]) + self.get_params( + "basic", + ["version", "health_check", "debug", "timeout"] + ) self.get_params("lnet", ["peers", "routes", "routers", "nis"]) + self.get_params( + "ldlm-lru", + ["ldlm.namespaces.*.lru_max_age", "ldlm.namespaces.*.lru_size"] + ) self.get_params("ldlm-states", ["*.*.state"]) self.get_params("jobid", ["jobid_name", "jobid_var"]) self.get_params("job-stats", ["*.*.job_stats"]) - self.get_params("exports", ["*.*.exports.*.*"]) + self.get_params("server_uuids", ["*.*.*server_uuid"]) + self.get_params("mgc_irstate", ["mgc.*.ir_state"]) # Client Specific self.add_cmd_output([ "lfs df", "lfs df -i" ]) + self.get_params("osc_client", [ + "osc.*.max_dirty_mb", + "osc.*.max_pages_per_rpc", + "osc.*.checksums", + "osc.*.max_rpcs_in_flight" + ]) # Server Specific self.get_params("osd", ["osd-*.*.{mntdev,files*," + "kbytes*,blocksize,brw_stats}"]) self.get_params("quota", ["osd-*.*.quota_slave." + "{info,limit_*,acct_*}"]) + self.get_params("mgs", ["mgs.MGS.ir_timeout", "mgs.MGS.live.*"]) + self.get_params("exports", ["*.*.exports.*.*"]) + self.get_params("mntdev", ["osd*.*.mntdev"]) + # mb_groups can be VERY large, and provide minimal debug usefulness + self.add_forbidden_path("*/mb_groups") self.add_copy_spec([ "/sys/fs/ldiskfs", "/proc/fs/ldiskfs", diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/lxd.py sosreport-4.5.6/sos/report/plugins/lxd.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/lxd.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/lxd.py 2023-07-20 21:38:49.000000000 +0000 @@ -17,7 +17,7 @@ plugin_name = 'lxd' profiles = ('container',) packages = ('lxd',) - commands = ('lxd',) + commands = ('lxc', 'lxd',) def setup(self): @@ -41,8 +41,8 @@ lxd_pred = SoSPredicate(self, kmods=lxd_kmods, required={'kmods': 'all'}) - snap_list = self.exec_cmd('snap list lxd') - if snap_list["status"] == 0: + lxd_pkg = self.policy.package_manager.pkg_by_name('lxd') + if lxd_pkg and lxd_pkg['pkg_manager'] == 'snap': self.add_cmd_output("lxd.buginfo", pred=lxd_pred) self.add_copy_spec([ diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/maas.py sosreport-4.5.6/sos/report/plugins/maas.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/maas.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/maas.py 2023-07-20 21:38:49.000000000 +0000 @@ -57,14 +57,10 @@ return ret['status'] == 0 def _is_snap_installed(self): - return self.exec_cmd('snap list maas')["status"] == 0 - - def check_enabled(self): - if super().check_enabled(): - # deb-based MAAS and existing triggers - return True - # Do we have the snap installed? - return self._is_snap_installed() + maas_pkg = self.policy.package_manager.pkg_by_name('maas') + if maas_pkg: + return maas_pkg['pkg_manager'] == 'snap' + return False def setup(self): self._is_snap = self._is_snap_installed() @@ -91,7 +87,9 @@ "/etc/squid-deb-proxy", "/etc/maas", "/var/lib/maas/dhcp*", - "/var/log/apache2*", + "/var/lib/maas/http/*.conf", + "/var/lib/maas/*.conf", + "/var/lib/maas/rsyslog", "/var/log/maas*", "/var/log/upstart/maas-*", ]) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/microshift.py sosreport-4.5.6/sos/report/plugins/microshift.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/microshift.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/microshift.py 2023-07-20 21:38:49.000000000 +0000 @@ -26,8 +26,10 @@ short_desc = 'Microshift' plugin_name = 'microshift' plugin_timeout = 900 - packages = ('microshift', 'microshift-selinux', 'microshift-networking') - services = (plugin_name,) + packages = ('microshift', 'microshift-selinux', 'microshift-networking', + 'microshift-greenboot') + services = (plugin_name, 'greenboot-healthcheck', + 'greenboot-task-runner', 'redboot-task-runner') profiles = (plugin_name,) localhost_kubeconfig = '/var/lib/microshift/resources/kubeadmin/kubeconfig' @@ -146,7 +148,10 @@ Output format for this function is based on `oc adm inspect` command, which is used to retrieve all API resources from the cluster. """ - self.add_forbidden_path('/var/lib/microshift') + if self.path_exists('/var/lib/microshift-backups'): + self.add_copy_spec(['/var/lib/microshift-backups/*/version', + '/var/lib/microshift-backups/*.json']) + self.add_cmd_output([ 'microshift version', 'microshift show-config -m effective' diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openssl.py sosreport-4.5.6/sos/report/plugins/openssl.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openssl.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openssl.py 2023-07-20 21:38:49.000000000 +0000 @@ -27,7 +27,7 @@ "challengePassword" ] - regexp = r"(?m)^(\s*#?\s*(%s).*=)(.*)" % "|".join(protect_keys) + regexp = r"^(\s*#?\s*(%s).*=)(.*)" % "|".join(protect_keys) self.do_file_sub( '/etc/ssl/openssl.cnf', diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_ansible.py sosreport-4.5.6/sos/report/plugins/openstack_ansible.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_ansible.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_ansible.py 2023-07-20 21:38:49.000000000 +0000 @@ -34,7 +34,7 @@ "/etc/openstack_deploy/user_secrets.yml", "/etc/rpc_deploy/user_secrets.yml" ] - regexp = r"(?m)^\s*#*([\w_]*:\s*).*" + regexp = r"^\s*#*([\w_]*:\s*).*" for secrets_file in secrets_files: self.do_path_regex_sub( secrets_file, diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_aodh.py sosreport-4.5.6/sos/report/plugins/openstack_aodh.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_aodh.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_aodh.py 2023-07-20 21:38:49.000000000 +0000 @@ -80,11 +80,11 @@ connection_keys = ["connection", "backend_url", "transport_url"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_barbican.py sosreport-4.5.6/sos/report/plugins/openstack_barbican.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_barbican.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_barbican.py 2023-07-20 21:38:49.000000000 +0000 @@ -42,7 +42,7 @@ ] self.do_file_sub( "/etc/barbican/barbican.conf", - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1********" ) @@ -50,7 +50,7 @@ self.do_path_regex_sub( "/etc/barbican/barbican.conf", - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6") diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_ceilometer.py sosreport-4.5.6/sos/report/plugins/openstack_ceilometer.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_ceilometer.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_ceilometer.py 2023-07-20 21:38:49.000000000 +0000 @@ -59,11 +59,11 @@ connection_keys = ["connection", "backend_url", "transport_url"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_cinder.py sosreport-4.5.6/sos/report/plugins/openstack_cinder.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_cinder.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_cinder.py 2023-07-20 21:38:49.000000000 +0000 @@ -94,11 +94,11 @@ connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_designate.py sosreport-4.5.6/sos/report/plugins/openstack_designate.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_designate.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_designate.py 2023-07-20 21:38:49.000000000 +0000 @@ -81,7 +81,7 @@ "ssl_key_password", "ssl_client_key_password", "memcache_secret_key" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub("/etc/designate/*", regexp, r"\1*********") self.do_path_regex_sub( diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_glance.py sosreport-4.5.6/sos/report/plugins/openstack_glance.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_glance.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_glance.py 2023-07-20 21:38:49.000000000 +0000 @@ -96,11 +96,11 @@ connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_heat.py sosreport-4.5.6/sos/report/plugins/openstack_heat.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_heat.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_heat.py 2023-07-20 21:38:49.000000000 +0000 @@ -109,11 +109,11 @@ connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_horizon.py sosreport-4.5.6/sos/report/plugins/openstack_horizon.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_horizon.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_horizon.py 2023-07-20 21:38:49.000000000 +0000 @@ -50,7 +50,7 @@ "SECRET_KEY", "EMAIL_HOST_PASSWORD" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) for regpath in [r"/etc/openstack-dashboard/.*\.json", "/etc/openstack-dashboard/local_settings$"]: self.do_path_regex_sub(regpath, regexp, r"\1*********") diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_instack.py sosreport-4.5.6/sos/report/plugins/openstack_instack.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_instack.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_instack.py 2023-07-20 21:38:49.000000000 +0000 @@ -133,13 +133,13 @@ "undercloud_swift_password", "undercloud_tuskar_password", ] - regexp = r"((?m)(%s)=)(.*)" % "|".join(protected_keys) + regexp = r"((%s)=)(.*)" % "|".join(protected_keys) self.do_file_sub("/home/stack/.instack/install-undercloud.log", regexp, r"\1*********") self.do_file_sub(UNDERCLOUD_CONF_PATH, regexp, r"\1*********") protected_json_keys = ["pm_password", "ssh-key", "password"] - json_regexp = r'((?m)"(%s)": )(".*?")' % "|".join(protected_json_keys) + json_regexp = r'("(%s)": )(".*?")' % "|".join(protected_json_keys) self.do_file_sub("/home/stack/instackenv.json", json_regexp, r"\1*********") self.do_file_sub('/home/stack/.tripleo/history', diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_ironic.py sosreport-4.5.6/sos/report/plugins/openstack_ironic.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_ironic.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_ironic.py 2023-07-20 21:38:49.000000000 +0000 @@ -145,11 +145,11 @@ connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_keystone.py sosreport-4.5.6/sos/report/plugins/openstack_keystone.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_keystone.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_keystone.py 2023-07-20 21:38:49.000000000 +0000 @@ -98,11 +98,11 @@ connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) @@ -110,7 +110,7 @@ # obfuscate LDAP plaintext passwords in domain config dir self.do_path_regex_sub( self.domain_config_dir, - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1********" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_manila.py sosreport-4.5.6/sos/report/plugins/openstack_manila.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_manila.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_manila.py 2023-07-20 21:38:49.000000000 +0000 @@ -64,11 +64,11 @@ connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_neutron.py sosreport-4.5.6/sos/report/plugins/openstack_neutron.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_neutron.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_neutron.py 2023-07-20 21:38:49.000000000 +0000 @@ -89,11 +89,11 @@ connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_nova.py sosreport-4.5.6/sos/report/plugins/openstack_nova.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_nova.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_nova.py 2023-07-20 21:38:49.000000000 +0000 @@ -109,6 +109,7 @@ self.add_copy_spec(self.path_join(novadir, novalog)) self.add_copy_spec([ "/var/log/{}*/nova*.log".format(self.apachepkg), + "/var/log/{}*/placement*.log".format(self.apachepkg), ]) pp = ['', '_libvirt', '_metadata', '_placement'] @@ -154,11 +155,11 @@ connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_octavia.py sosreport-4.5.6/sos/report/plugins/openstack_octavia.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_octavia.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_octavia.py 2023-07-20 21:38:49.000000000 +0000 @@ -117,7 +117,7 @@ "connection", "transport_url", "server_certs_key_passphrase", "memcache_secret_key" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub("/etc/octavia/*", regexp, r"\1*********") self.do_path_regex_sub( diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_placement.py sosreport-4.5.6/sos/report/plugins/openstack_placement.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_placement.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_placement.py 2023-07-20 21:38:49.000000000 +0000 @@ -75,11 +75,11 @@ connection_keys = ["database_connection", "slave_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_sahara.py sosreport-4.5.6/sos/report/plugins/openstack_sahara.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_sahara.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_sahara.py 2023-07-20 21:38:49.000000000 +0000 @@ -52,11 +52,11 @@ connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_swift.py sosreport-4.5.6/sos/report/plugins/openstack_swift.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_swift.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_swift.py 2023-07-20 21:38:49.000000000 +0000 @@ -62,11 +62,11 @@ connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_trove.py sosreport-4.5.6/sos/report/plugins/openstack_trove.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/openstack_trove.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/openstack_trove.py 2023-07-20 21:38:49.000000000 +0000 @@ -51,11 +51,11 @@ connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ovirt.py sosreport-4.5.6/sos/report/plugins/ovirt.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/ovirt.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/ovirt.py 2023-07-20 21:38:49.000000000 +0000 @@ -263,7 +263,7 @@ "pool.default.ssl.truststore.password", "config.datasource.dbpassword" ] - regexp = r"((?m)^\s*#*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*#*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub(r"/etc/ovirt-engine/aaa/.*\.properties", regexp, r"\1*********") diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/saltmaster.py sosreport-4.5.6/sos/report/plugins/saltmaster.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/report/plugins/saltmaster.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/report/plugins/saltmaster.py 2023-07-20 21:38:49.000000000 +0000 @@ -29,7 +29,7 @@ self.add_cmd_output("salt-key --list all") def postproc(self): - regexp = r'((?m)^\s+.*(pass|secret|(? +# +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, UbuntuPlugin + + +class Vault(Plugin, UbuntuPlugin): + """The Vault plugin collects the current status of the vault + snap on a Ubuntu machine. + + It will collect logs from journal, vault status and configuration + """ + + short_desc = 'Manage access to secrets and protect sensitive data' + + plugin_name = 'vault' + profiles = ('sysmgmt', 'security') + services = ('vault',) + package = ('vault',) + + def setup(self): + + vault_cfg = "/var/snap/vault/common/vault.hcl" + + self.add_copy_spec(vault_cfg) + + try: + with open(vault_cfg, 'r') as cf: + for line in cf.read().splitlines(): + if not line: + continue + words = line.split('=') + if words[0].strip() == 'api_addr': + api_addr = words[1].strip('\" ') + self.add_cmd_output("vault status", + env={'VAULT_ADDR': api_addr}) + except IOError as error: + self._log_error('Could not open conf file %s: %s' % + (vault_cfg, error)) + + def postproc(self): + self.do_file_sub( + "/var/snap/vault/common/vault.hcl", + r"(password\s?=\s?).*", + r"\1******" + ) + +# vim: set et ts=4 sw=4 : diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos/utilities.py sosreport-4.5.6/sos/utilities.py --- sosreport-4.5.4ubuntu0.23.04.1/sos/utilities.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos/utilities.py 2023-07-20 21:38:49.000000000 +0000 @@ -319,6 +319,8 @@ def path_exists(path, sysroot): + if '*' in path: + return _os_wrapper(path, sysroot, 'glob', module=glob) return _os_wrapper(path, sysroot, 'exists') diff -Nru sosreport-4.5.4ubuntu0.23.04.1/sos.spec sosreport-4.5.6/sos.spec --- sosreport-4.5.4ubuntu0.23.04.1/sos.spec 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/sos.spec 2023-07-20 21:38:49.000000000 +0000 @@ -1,22 +1,23 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos -Version: 4.5.4 +Version: 4.5.6 Release: 1%{?dist} -Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{name}-%{version}.tar.gz License: GPL-2.0-or-later -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot BuildArch: noarch -Url: https://github.com/sosreport/sos/ +Url: https://github.com/sosreport/sos BuildRequires: python3-devel BuildRequires: python3-setuptools -BuildRequires: gettext Requires: python3-rpm Requires: python3-pexpect Requires: python3-setuptools Recommends: python3-magic +# Mandatory just for uploading to a SFTP server: +Recommends: python3-requests Recommends: python3-pyyaml Obsoletes: sos-collector <= 1.9 +# For the _tmpfilesdir macro. +BuildRequires: systemd %description Sos is a set of tools that gathers information about system @@ -27,36 +28,56 @@ %prep %setup -qn %{name}-%{version} +%if 0%{?fedora} >= 39 +%generate_buildrequires +%pyproject_buildrequires +%endif + %build +%if 0%{?fedora} >= 39 +%pyproject_wheel +%else %py3_build +%endif %install +%if 0%{?fedora} >= 39 +%pyproject_install +%pyproject_save_files sos +%else %py3_install '--install-scripts=%{_sbindir}' +%endif -install -d -m 755 ${RPM_BUILD_ROOT}/etc/sos -install -d -m 700 ${RPM_BUILD_ROOT}/etc/sos/cleaner -install -d -m 755 ${RPM_BUILD_ROOT}/etc/sos/presets.d -install -d -m 755 ${RPM_BUILD_ROOT}/etc/sos/groups.d -install -d -m 755 ${RPM_BUILD_ROOT}/etc/sos/extras.d -install -d -m 755 ${RPM_BUILD_ROOT}/etc/tmpfiles.d/ -install -m 644 %{name}.conf ${RPM_BUILD_ROOT}/etc/sos/%{name}.conf -install -m 644 tmpfiles/tmpfilesd-sos-rh.conf ${RPM_BUILD_ROOT}/etc/tmpfiles.d/%{name}.conf +install -d -m 755 %{buildroot}%{_sysconfdir}/%{name} +install -d -m 700 %{buildroot}%{_sysconfdir}/%{name}/cleaner +install -d -m 755 %{buildroot}%{_sysconfdir}/%{name}/presets.d +install -d -m 755 %{buildroot}%{_sysconfdir}/%{name}/groups.d +install -d -m 755 %{buildroot}%{_sysconfdir}/%{name}/extras.d +install -d -m 755 %{buildroot}%{_tmpfilesdir} +install -m 644 %{name}.conf %{buildroot}%{_sysconfdir}/%{name}/%{name}.conf +install -m 644 tmpfiles/tmpfilesd-sos-rh.conf %{buildroot}%{_tmpfilesdir}/%{name}.conf -rm -rf ${RPM_BUILD_ROOT}/usr/config/ +rm -rf %{buildroot}/usr/config/ %find_lang %{name} || echo 0 # internationalization is currently broken. Uncomment this line once fixed. # %%files -f %%{name}.lang %files +%if 0%{?fedora} >= 39 +%{_bindir}/sos +%{_bindir}/sosreport +%{_bindir}/sos-collector +%else %{_sbindir}/sos %{_sbindir}/sosreport %{_sbindir}/sos-collector +%endif %dir /etc/sos/cleaner %dir /etc/sos/presets.d %dir /etc/sos/extras.d %dir /etc/sos/groups.d -/etc/tmpfiles.d/%{name}.conf +%{_tmpfilesdir}/%{name}.conf %{python3_sitelib}/* %{_mandir}/man1/* %{_mandir}/man5/* @@ -65,6 +86,12 @@ %config(noreplace) %{_sysconfdir}/sos/sos.conf %changelog +* Thu Jul 20 2023 Jake Hunsaker = 4.5.6 +- New upstream release + +* Fri Jun 23 2023 Jake Hunsaker = 4.5.5 +- New upstream release + * Fri May 26 2023 Jake Hunsaker = 4.5.4 - New upstream release diff -Nru sosreport-4.5.4ubuntu0.23.04.1/tests/cleaner_tests/basic_function_tests/report_with_mask.py sosreport-4.5.6/tests/cleaner_tests/basic_function_tests/report_with_mask.py --- sosreport-4.5.4ubuntu0.23.04.1/tests/cleaner_tests/basic_function_tests/report_with_mask.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/tests/cleaner_tests/basic_function_tests/report_with_mask.py 2023-07-20 21:38:49.000000000 +0000 @@ -9,6 +9,7 @@ from sos_tests import StageOneReportTest, StageTwoReportTest import re +from os import stat class ReportWithMask(StageOneReportTest): @@ -18,6 +19,17 @@ """ sos_cmd = '--mask -o host,networking' + hosts_obfuscated = None + + def pre_sos_setup(self): + # obfuscate a random word from /etc/hosts and ensure the updated + # sanitised file has same permissions (a+r) + try: + self.hosts_obfuscated = open('/etc/hosts').read().strip('#\n').split()[-1] + except (FileNotFoundError, IndexError) as e: + self.warning(f"Unable to process /etc/hosts: {e}") + if self.hosts_obfuscated: + self.sos_cmd += f' --keywords={self.hosts_obfuscated}' def test_mask_was_run(self): self.assertOutputContains('Beginning obfuscation') @@ -53,6 +65,12 @@ mac = line.strip().split()[1] assert mac.startswith('53:4f:53'), "Found unobfuscated mac addr %s" % mac + def test_perms_unchanged_on_modified_file(self): + if self.hosts_obfuscated: + imode_orig = stat('/etc/hosts').st_mode + imode_obfuscated = stat(self.get_name_in_archive('etc/hosts')).st_mode + self.assertEqual(imode_orig, imode_obfuscated) + class ReportWithCleanedKeywords(StageOneReportTest): """Testing for obfuscated keywords provided by the user diff -Nru sosreport-4.5.4ubuntu0.23.04.1/tests/report_tests/options_tests/options_tests.py sosreport-4.5.6/tests/report_tests/options_tests/options_tests.py --- sosreport-4.5.4ubuntu0.23.04.1/tests/report_tests/options_tests/options_tests.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/tests/report_tests/options_tests/options_tests.py 2023-07-20 21:38:49.000000000 +0000 @@ -22,8 +22,8 @@ def test_case_id_from_config(self): self.assertTrue('8675309' in self.archive) - def test_plugins_skipped_from_config(self): - self.assertPluginNotIncluded(['networking', 'logs']) + def test_plugins_only_from_config(self): + self.assertOnlyPluginsIncluded(['host', 'kernel']) def test_plugopts_logged_from_config(self): self.assertSosLogContains( @@ -41,5 +41,5 @@ def test_effective_options_logged_correctly(self): self.assertSosLogContains( - "effective options now: --batch --case-id 8675309 --plugopts kernel.with-timer=on,kernel.trace=yes --skip-plugins networking,logs" + "effective options now: --batch --case-id 8675309 --only-plugins host,kernel --plugopts kernel.with-timer=on,kernel.trace=yes" ) diff -Nru sosreport-4.5.4ubuntu0.23.04.1/tests/report_tests/options_tests/options_tests_sos.conf sosreport-4.5.6/tests/report_tests/options_tests/options_tests_sos.conf --- sosreport-4.5.4ubuntu0.23.04.1/tests/report_tests/options_tests/options_tests_sos.conf 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/tests/report_tests/options_tests/options_tests_sos.conf 2023-07-20 21:38:49.000000000 +0000 @@ -2,7 +2,7 @@ #verbose = 3 [report] -skip-plugins = networking,logs +only-plugins = host,kernel case-id = 8675309 [collect] diff -Nru sosreport-4.5.4ubuntu0.23.04.1/tests/unittests/cleaner_tests.py sosreport-4.5.6/tests/unittests/cleaner_tests.py --- sosreport-4.5.4ubuntu0.23.04.1/tests/unittests/cleaner_tests.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/tests/unittests/cleaner_tests.py 2023-07-20 21:38:49.000000000 +0000 @@ -20,7 +20,11 @@ from sos.cleaner.mappings.hostname_map import SoSHostnameMap from sos.cleaner.mappings.keyword_map import SoSKeywordMap from sos.cleaner.mappings.ipv6_map import SoSIPv6Map - +from sos.cleaner.preppers import SoSPrepper +from sos.cleaner.preppers.hostname import HostnamePrepper +from sos.cleaner.preppers.ip import IPPrepper +from sos.cleaner.archives.sos import SoSReportArchive +from sos.options import SoSOptions class CleanerMapTests(unittest.TestCase): @@ -28,7 +32,7 @@ self.mac_map = SoSMacMap() self.ip_map = SoSIPMap() self.host_map = SoSHostnameMap() - self.host_map.load_domains_from_options(['redhat.com']) + self.host_map.sanitize_item('redhat.com') self.kw_map = SoSKeywordMap() self.ipv6_map = SoSIPv6Map() @@ -152,13 +156,14 @@ self.ip_parser = SoSIPParser(config={}) self.ipv6_parser = SoSIPv6Parser(config={}) self.mac_parser = SoSMacParser(config={}) - self.host_parser = SoSHostnameParser(config={}, - opt_domains=['foobar.com']) - self.kw_parser = SoSKeywordParser(config={}, keywords=['foobar']) + self.host_parser = SoSHostnameParser(config={}) + self.host_parser.mapping.add('foobar.com') + self.kw_parser = SoSKeywordParser(config={}) + self.kw_parser.mapping.add('foobar') self.kw_parser_none = SoSKeywordParser(config={}) self.kw_parser.generate_item_regexes() - self.uname_parser = SoSUsernameParser(config={}, - opt_names=['DOMAIN\myusername']) + self.uname_parser = SoSUsernameParser(config={}) + self.uname_parser.mapping.add('DOMAIN\myusername') def test_ip_parser_valid_ipv4_line(self): line = 'foobar foo 10.0.0.1/24 barfoo bar' @@ -210,22 +215,22 @@ def test_hostname_load_hostname_string(self): fqdn = 'myhost.subnet.example.com' - self.host_parser.load_hostname_into_map(fqdn) + self.host_parser.mapping.add(fqdn) def test_hostname_valid_domain_line(self): - self.host_parser.load_hostname_into_map('myhost.subnet.example.com') + self.host_parser.mapping.add('myhost.subnet.example.com') line = 'testing myhost.subnet.example.com in a string' _test = self.host_parser.parse_line(line)[0] self.assertNotEqual(line, _test) def test_hostname_short_name_in_line(self): - self.host_parser.load_hostname_into_map('myhost.subnet.example.com') + self.host_parser.mapping.add('myhost.subnet.example.com') line = 'testing just myhost in a line' _test = self.host_parser.parse_line(line)[0] self.assertNotEqual(line, _test) def test_obfuscate_whole_fqdn_for_given_domainname(self): - self.host_parser.load_hostname_into_map('sostestdomain.domain') + self.host_parser.mapping.add('sostestdomain.domain') line = 'let obfuscate soshost.sostestdomain.domain' _test = self.host_parser.parse_line(line)[0] self.assertFalse('soshost' in _test) @@ -274,3 +279,35 @@ line = "DOMAIN\myusername" _test = self.uname_parser.parse_line(line)[0] self.assertNotEqual(line, _test) + + +class PrepperTests(unittest.TestCase): + """ + Ensure that the translations for different parser/mapping methods are + working + """ + + def setUp(self): + self.prepper = SoSPrepper(SoSOptions()) + self.archive = SoSReportArchive( + archive_path='tests/test_data/sosreport-cleanertest-2021-08-03-qpkxdid.tar.xz', + tmpdir='/tmp' + ) + self.host_prepper = HostnamePrepper(SoSOptions(domains=[])) + self.ipv4_prepper = IPPrepper(SoSOptions()) + + def test_parser_method_translation(self): + self.assertEqual([], self.prepper.get_parser_file_list('hostname', None)) + + def test_mapping_method_translation(self): + self.assertEqual([], self.prepper.get_items_for_map('foobar', None)) + + def test_hostname_prepper_map_items(self): + self.assertEqual(['cleanertest'], self.host_prepper.get_items_for_map('hostname', self.archive)) + + def test_ipv4_prepper_parser_files(self): + self.assertEqual(['sos_commands/networking/ip_-o_addr'], self.ipv4_prepper.get_parser_file_list('ip', self.archive)) + + def test_ipv4_prepper_invalid_parser_files(self): + self.assertEqual([], self.ipv4_prepper.get_parser_file_list('foobar', self.archive)) + diff -Nru sosreport-4.5.4ubuntu0.23.04.1/tests/unittests/plugin_tests.py sosreport-4.5.6/tests/unittests/plugin_tests.py --- sosreport-4.5.4ubuntu0.23.04.1/tests/unittests/plugin_tests.py 2023-05-26 17:02:49.000000000 +0000 +++ sosreport-4.5.6/tests/unittests/plugin_tests.py 2023-07-20 21:38:49.000000000 +0000 @@ -45,7 +45,7 @@ def name(self): return "mock.archive" - def add_file(self, src, dest=None): + def add_file(self, src, dest=None, force=False): if not dest: dest = src self.m[src] = dest