diff -Nru click-reviewers-tools-0.27/clickreviews/cr_bin_path.py click-reviewers-tools-0.28/clickreviews/cr_bin_path.py --- click-reviewers-tools-0.27/clickreviews/cr_bin_path.py 2015-05-04 10:49:48.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_bin_path.py 2015-05-27 22:24:37.000000000 +0000 @@ -42,6 +42,8 @@ self.bin_paths = dict() if self.is_snap and 'binaries' in self.pkg_yaml: + if len(self.pkg_yaml['binaries']) == 0: + error("package.yaml malformed: 'binaries' is empty") for binary in self.pkg_yaml['binaries']: if 'name' not in binary: error("package.yaml malformed: required 'name' not found " diff -Nru click-reviewers-tools-0.27/clickreviews/cr_common.py click-reviewers-tools-0.28/clickreviews/cr_common.py --- click-reviewers-tools-0.27/clickreviews/cr_common.py 2015-05-04 10:49:48.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_common.py 2015-05-27 22:47:53.000000000 +0000 @@ -33,6 +33,7 @@ DEBUGGING = False UNPACK_DIR = None +RAW_UNPACK_DIR = None # cleanup import atexit @@ -42,6 +43,9 @@ global UNPACK_DIR if UNPACK_DIR is not None and os.path.isdir(UNPACK_DIR): recursive_rm(UNPACK_DIR) + global RAW_UNPACK_DIR + if RAW_UNPACK_DIR is not None and os.path.isdir(RAW_UNPACK_DIR): + recursive_rm(RAW_UNPACK_DIR) atexit.register(cleanup_unpack) @@ -89,7 +93,6 @@ "frameworks", "icon", "immutable-config", - "integration", "oem", "services", "source", @@ -126,6 +129,11 @@ UNPACK_DIR = unpack_click(fn) self.unpack_dir = UNPACK_DIR + global RAW_UNPACK_DIR + if RAW_UNPACK_DIR is None: + RAW_UNPACK_DIR = raw_unpack_pkg(fn) + self.raw_unpack_dir = RAW_UNPACK_DIR + # Get some basic information from the control file control_file = self._extract_control_file() tmp = list(Deb822.iter_paragraphs(control_file)) @@ -209,6 +217,29 @@ return None # snappy packaging is still optional return open_file_read(y) + def _extract_hashes_yaml(self): + '''Extract and read the snappy hashes.yaml''' + y = os.path.join(self.unpack_dir, "DEBIAN/hashes.yaml") + return open_file_read(y) + + def _extract_statinfo(self, fn): + '''Extract statinfo from file''' + try: + st = os.stat(fn) + except Exception: + return None + return st + + def _path_join(self, dirname, rest): + return os.path.join(dirname, rest) + + def _get_sha512sum(self, fn): + '''Get sha512sum of file''' + (rc, out) = cmd(['sha512sum', fn]) + if rc != 0: + return None + return out.split()[0] + def _check_path_exists(self): '''Check that the provided path exists''' if not os.path.exists(self.click_package): @@ -328,9 +359,6 @@ elif f in ["binaries", "services"] and not \ isinstance(self.pkg_yaml[f], list): error("yaml malformed: '%s' is not list:\n%s" % (f, yp)) - elif f == "integration" and not isinstance(self.pkg_yaml[f], - dict): - error("yaml malformed: '%s' is not dict:\n%s" % (f, yp)) elif f in ["icon", "source", "type", "vendor"] and not \ isinstance(self.pkg_yaml[f], str): error("yaml malformed: '%s' is not str:\n%s" % (f, yp)) @@ -379,7 +407,12 @@ def _verify_pkgname(self, n): '''Verify package name''' - if re.search(r'^[a-z0-9][a-z0-9+.-]+$', n): + if self.is_snap: + # snaps can't have '.' in the name + pat = re.compile(r'^[a-z0-9][a-z0-9+-]+$') + else: + pat = re.compile(r'^[a-z0-9][a-z0-9+.-]+$') + if pat.search(n): return True return False @@ -606,6 +639,37 @@ if dest is None: dest = d + else: + shutil.move(d, dest) + + return dest + + +def raw_unpack_pkg(fn, dest=None): + '''Unpack raw package''' + if not os.path.isfile(fn): + error("Could not find '%s'" % fn) + pkg = fn + if not pkg.startswith('/'): + pkg = os.path.abspath(pkg) + + if dest is not None and os.path.exists(dest): + error("'%s' exists. Aborting." % dest) + + d = tempfile.mkdtemp(prefix='review-') + + curdir = os.getcwd() + os.chdir(d) + (rc, out) = cmd(['ar', 'x', pkg]) + os.chdir(curdir) + + if rc != 0: + if os.path.isdir(d): + recursive_rm(d) + error("'ar x' failed with '%d':\n%s" % (rc, out)) + + if dest is None: + dest = d else: shutil.move(d, dest) diff -Nru click-reviewers-tools-0.27/clickreviews/cr_desktop.py click-reviewers-tools-0.28/clickreviews/cr_desktop.py --- click-reviewers-tools-0.27/clickreviews/cr_desktop.py 2015-04-30 20:32:50.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_desktop.py 2015-05-22 15:14:18.000000000 +0000 @@ -214,6 +214,8 @@ def check_desktop_exec_webapp_container(self): '''Check Exec=webapp-container entry''' + fwk = self.manifest['framework'] + for app in sorted(self.desktop_entries): de = self._get_desktop_entry(app) t = 'info' @@ -224,6 +226,19 @@ s = "missing key 'Exec'" self._add_result(t, n, s) continue + elif de.getExec().split()[0] == "ubuntu-html5-app-launcher" and \ + fwk.startswith('ubuntu-sdk') and not \ + (fwk.startswith('ubuntu-sdk-13') or + fwk.startswith('ubuntu-sdk-14')): + # ubuntu-html5-app-launcher only available in ubuntu-sdk-14.10 + # and lower + t = 'error' + s = "ubuntu-html5-app-launcher is obsoleted in 15.04 " + \ + "frameworks and higher. Please use 'webapp-container' " + \ + "instead and ensure your security policy uses the " + \ + "'ubuntu-webapp' template" + self._add_result(t, n, s) + continue elif de.getExec().split()[0] != "webapp-container": s = "SKIPPED (not webapp-container)" self._add_result(t, n, s) diff -Nru click-reviewers-tools-0.27/clickreviews/cr_framework.py click-reviewers-tools-0.28/clickreviews/cr_framework.py --- click-reviewers-tools-0.27/clickreviews/cr_framework.py 2015-04-30 20:32:50.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_framework.py 2015-05-18 21:08:42.000000000 +0000 @@ -66,12 +66,12 @@ def _extract_framework_policy(self): '''Get framework policy files''' policy_dict = dict() - unknown_dict = dict() + unknown = [] fpdir = os.path.join(self.unpack_dir, "meta", "framework-policy") for i in glob.glob("%s/*" % fpdir): rel_i = os.path.basename(i) if not os.path.isdir(i) or rel_i not in self.framework_policy_dirs: - unknown_dict.append(os.path.relpath(i, self.unpack_dir)) + unknown.append(os.path.relpath(i, self.unpack_dir)) continue policy_dict[rel_i] = dict() @@ -79,21 +79,20 @@ rel_j = os.path.basename(j) if not os.path.isdir(j) or \ rel_j not in self.framework_policy_subdirs: - unknown_dict.append(os.path.relpath(j, self.unpack_dir)) + unknown.append(os.path.relpath(j, self.unpack_dir)) continue policy_dict[rel_i][rel_j] = dict() for k in glob.glob("%s/*" % j): rel_k = os.path.basename(k) if not os.path.isfile(k): - unknown_dict.append(os.path.relpath(k, - self.unpack_dir)) + unknown.append(os.path.relpath(k, self.unpack_dir)) continue fh = open_file_read(k) policy_dict[rel_i][rel_j][rel_k] = fh.read() fh.close() - return (policy_dict, unknown_dict) + return (policy_dict, unknown) def _has_framework_in_metadir(self): '''Check if snap has meta/.framework''' @@ -223,7 +222,7 @@ if j not in self.framework_policy[other] or \ k not in self.framework_policy[other][j]: t = 'error' - s = "Could not find mathcing '%s/%s/%s'" % (other, + s = "Could not find matching '%s/%s/%s'" % (other, j, k) self._add_result(t, n, s) diff -Nru click-reviewers-tools-0.27/clickreviews/cr_lint.py click-reviewers-tools-0.28/clickreviews/cr_lint.py --- click-reviewers-tools-0.27/clickreviews/cr_lint.py 2015-05-04 10:49:48.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_lint.py 2015-06-08 16:06:07.000000000 +0000 @@ -20,9 +20,11 @@ import glob import os import re +import stat +import yaml from clickreviews.frameworks import Frameworks -from clickreviews.cr_common import ClickReview, open_file_read, cmd +from clickreviews.cr_common import ClickReview, open_file_read, cmd, error CONTROL_FILE_NAMES = ["control", "manifest", "preinst"] MINIMUM_CLICK_FRAMEWORK_VERSION = "0.4" @@ -251,7 +253,6 @@ s = "Maintainer=%s does not match manifest maintainer=%s" % \ (control['Maintainer'], self.manifest['maintainer']) else: - t = 'warn' s = 'Skipped: maintainer not in manifest' self._add_result(t, n, s) @@ -495,7 +496,9 @@ self._add_result(t, n, s) def check_pkgname(self): - '''Check package name valid''' + '''Check click package name valid''' + if self.is_snap: + return p = self.manifest['name'] # http://www.debian.org/doc/debian-policy/ch-controlfields.html t = 'info' @@ -572,8 +575,13 @@ n = 'maintainer_present' s = 'OK' if 'maintainer' not in self.manifest: - s = 'required maintainer field not specified in manifest' - self._add_result('error', n, s) + if self.is_snap: + s = 'Skipped optional maintainer field not specified in ' + \ + 'manifest' + else: + t = 'error' + s = 'required maintainer field not specified in manifest' + self._add_result(t, n, s) return self._add_result(t, n, s) @@ -1031,7 +1039,7 @@ return contents def check_snappy_readme_md(self): - '''Check package architecture in package.yaml is valid''' + '''Check snappy readme.md''' if not self.is_snap: return @@ -1055,3 +1063,224 @@ t = 'warn' s = "meta/readme.md is too short" self._add_result(t, n, s) + + def _check_innerpath_executable(self, fn): + '''Check that the provided path exists and is executable''' + return os.access(fn, os.X_OK) + + def check_snappy_config(self): + '''Check snappy config''' + if not self.is_snap: + return + + fn = os.path.join(self.unpack_dir, 'meta/hooks/config') + if fn not in self.pkg_files: + return + + t = 'info' + n = 'snappy_config_hook_executable' + s = 'OK' + if not self._check_innerpath_executable(fn): + t = 'error' + s = 'meta/hooks/config is not executable' + self._add_result(t, n, s) + + def check_snappy_services_and_binaries(self): + '''Services and binaries should not overlap''' + if not self.is_snap: + return + for exe_t in ['binaries', 'services']: + if exe_t not in self.pkg_yaml: + break + if exe_t == 'binaries': + other_exe_t = 'services' + else: + other_exe_t = 'binaries' + + if other_exe_t not in self.pkg_yaml: + break + + for a in self.pkg_yaml[exe_t]: + if 'name' not in a: + continue + # handle 'exec' in binaries + app = os.path.basename(a['name']) + + t = 'info' + n = 'snappy_%s_in_%s' % (app, other_exe_t) + s = 'OK' + for other_a in self.pkg_yaml[other_exe_t]: + other_app = os.path.basename(other_a['name']) + if app == other_app: + t = 'error' + s = "'%s' in both 'services' and 'binaries'" % app + break + self._add_result(t, n, s) + + def check_snappy_hashes(self): + '''Check snappy hashes.yaml''' + if not self.is_snap: + return + + def _check_allowed_perms(mode, allowed): + '''Check that mode only uses allowed perms''' + for p in mode[1:]: + if p not in allowed: + return False + return True + + try: + hashes_yaml = yaml.safe_load(self._extract_hashes_yaml()) + except Exception: + error("Could not load hashes.yaml. Is it properly formatted?") + + if 'archive-sha512' not in hashes_yaml: + t = 'error' + n = 'hashes_archive-sha512_present' + s = "'archive-sha512' not found in hashes.yaml" + self._add_result(t, n, s) + return + + # verify the ar file + t = 'info' + n = 'hashes_archive-sha512_valid' + s = 'OK' + fn = self._path_join(self.raw_unpack_dir, 'data.tar.gz') + sum = self._get_sha512sum(fn) + if hashes_yaml['archive-sha512'] != sum: + t = 'error' + s = "hash mismatch: '%s' != '%s'" % (hashes_yaml['archive-sha512'], + sum) + self._add_result(t, n, s) + return + self._add_result(t, n, s) + + if 'files' not in hashes_yaml: + t = 'error' + n = 'hashes_files_present' + s = "'files' not found in hashes.yaml" + self._add_result(t, n, s) + return + + # verify the individual files + errors = [] + badsums = [] + hash_files = set([]) # used to check with extra files + for entry in hashes_yaml['files']: + if 'name' not in entry: + errors.append("'name' not found for entry '%s'" % entry) + continue + else: + hash_files.add(entry['name']) + + if 'mode' not in entry: + errors.append("'mode' not found for entry '%s'" % + entry['name']) + continue + elif len(entry['mode']) != 10: + errors.append("malformed mode '%s' for entry '%s'" % + (entry['mode'], entry['name'])) + continue + elif entry['mode'].startswith('d'): + if not _check_allowed_perms(entry['mode'], + ['r', 'w', 'x', '-']): + errors.append("unusual mode '%s' for entry '%s'" % + (entry['mode'], entry['name'])) + # world write shouldn't be allowed + if entry['mode'][-2] != '-': + errors.append("'%s' is world-writable" % entry['name']) + continue + elif entry['mode'].startswith('l'): + if entry['mode'] != "lrwxrwxrwx": + errors.append("unusual mode '%s' for entry '%s'" % + (entry['mode'], entry['name'])) + continue + elif not entry['mode'].startswith('f'): + # files and symlinks are ok, everything else is not + errors.append("illegal file mode '%s': '%s' for '%s'" % + (entry['mode'][0], entry['mode'], entry['name'])) + continue + elif 'size' not in entry: + errors.append("'size' not found for entry: %s" % entry['name']) + continue + elif 'sha512' not in entry: + errors.append("'sha512' not found for entry: %s" % + entry['name']) + continue + + fn = self._path_join(self.unpack_dir, entry['name']) + + # quick verify the size, if it is wrong, we don't have to do the + # sha512sum + statinfo = self._extract_statinfo(fn) + # file is missing + if statinfo is None: + errors.append("'%s' does not exist" % entry['name']) + continue + if entry['size'] != statinfo.st_size: + errors.append("size %d != %d for '%s'" % (entry['size'], + statinfo.st_size, + entry['name'])) + continue + + # check for weird perms + if not _check_allowed_perms(entry['mode'], ['r', 'w', 'x', '-']): + errors.append("unusual mode '%s' for entry '%s'" % + (entry['mode'], entry['name'])) + continue + # world write shouldn't be allowed + if entry['mode'][-2] != '-': + errors.append("mode '%s' for '%s' is world-writable" % + (entry['mode'], entry['name'])) + continue + # verify perms match what is in hashes.yaml + filemode = stat.filemode(statinfo.st_mode)[1:] + if entry['mode'][1:] != filemode: + errors.append("mode '%s' != '%s' for '%s'" % + (entry['mode'][1:], filemode, entry['name'])) + continue + + # ok, now all the cheap tests are done so we can check if we have a + # valid sha512sum + sum = self._get_sha512sum(fn) + if entry['sha512'] != sum: + badsums.append("'%s' != '%s' for '%s'" % (entry['sha512'], sum, + entry['name'])) + + t = 'info' + n = 'sha512sums' + s = 'OK' + if len(badsums) > 0: + t = 'error' + s = 'found bad checksums: %s' % ", ".join(badsums) + self._add_result(t, n, s) + + t = 'info' + n = 'file_mode' + s = 'OK' + if len(errors) > 0: + t = 'error' + s = 'found errors in hashes.yaml: %s' % ", ".join(errors) + self._add_result(t, n, s) + + # Now check for extra files + t = 'info' + n = 'hashes_extra_files' + s = 'OK' + self._add_result(t, n, s) + extra = [] + click_compat_files = set(["DEBIAN/hashes.yaml", + "DEBIAN/control", + "DEBIAN/preinst", + "DEBIAN/manifest" + ]) + for f in self.pkg_files: + fn = os.path.relpath(f, self.unpack_dir) + if fn not in hash_files and fn not in click_compat_files: + extra.append(fn) + + if len(extra) > 0: + t = 'error' + s = 'found extra files not listed in hashes.yaml: %s' % \ + ", ".join(extra) + self._add_result(t, n, s) diff -Nru click-reviewers-tools-0.27/clickreviews/cr_scope.py click-reviewers-tools-0.28/clickreviews/cr_scope.py --- click-reviewers-tools-0.27/clickreviews/cr_scope.py 2015-04-30 20:32:50.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_scope.py 2015-05-14 14:40:22.000000000 +0000 @@ -112,6 +112,7 @@ 'icon', 'idletimeout', 'invisible', + 'keywords', 'locationdataneeded', 'resultsttltype', 'scoperunner', diff -Nru click-reviewers-tools-0.27/clickreviews/cr_security.py click-reviewers-tools-0.28/clickreviews/cr_security.py --- click-reviewers-tools-0.27/clickreviews/cr_security.py 2015-05-05 12:47:37.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_security.py 2015-06-10 21:02:17.000000000 +0000 @@ -19,6 +19,7 @@ from clickreviews.cr_common import ClickReview, error, open_file_read import clickreviews.cr_common as cr_common import clickreviews.apparmor_policy as apparmor_policy +import copy import json import os @@ -68,8 +69,7 @@ 'comment', 'copyright', 'name'] - self.required_fields = ['policy_groups', - 'policy_version'] + self.required_fields = ['policy_version'] self.redflag_fields = ['abstractions', 'binary', 'read_path', @@ -147,6 +147,9 @@ self._extract_security_profile(app) self.security_apps_profiles.append(app) + # snappy + self.sec_skipped_types = ['oem'] # these don't need security items + def _override_framework_policies(self, overrides): # override major framework policies self.major_framework_policy.update(overrides) @@ -464,7 +467,27 @@ continue self._add_result(t, n, s) - if m['template'] not in self._get_templates(vendor, version): + found = False + if m['template'] in self._get_templates(vendor, version): + found = True + elif self.is_snap: + frameworks = [] + if 'framework' in self.pkg_yaml: + frameworks = [x.strip() for x in + self.pkg_yaml['framework'].split(',')] + elif 'frameworks' in self.pkg_yaml: + frameworks = self.pkg_yaml['frameworks'] + elif 'type' in self.pkg_yaml and \ + self.pkg_yaml['type'] == 'framework': + # frameworks may reference their own policy groups + frameworks.append(self.pkg_yaml['name']) + + for f in frameworks: + if m['template'].startswith("%s_" % f): + found = True + break + + if not found: t = 'error' s = "specified unsupported template '%s'" % m['template'] @@ -590,7 +613,7 @@ n = 'policy_groups_exists_%s (%s)' % (app, f) if 'policy_groups' not in m: # If template not specified, we just use the default - self._add_result('warn', n, 'no policy groups specified') + self._add_result('info', n, 'no policy groups specified') continue elif 'policy_version' not in m: self._add_result('error', n, @@ -625,6 +648,18 @@ s = 'duplicate policy groups found: %s' % ", ".join(tmp) self._add_result(t, n, s) + frameworks = [] + if self.is_snap: + if 'framework' in self.pkg_yaml: + frameworks = [x.strip() for x in + self.pkg_yaml['framework'].split(',')] + elif 'frameworks' in self.pkg_yaml: + frameworks = self.pkg_yaml['frameworks'] + elif 'type' in self.pkg_yaml and \ + self.pkg_yaml['type'] == 'framework': + # frameworks may reference their own policy groups + frameworks.append(self.pkg_yaml['name']) + # If we got here, we can see if valid policy groups were specified for i in m['policy_groups']: t = 'info' @@ -640,10 +675,20 @@ continue found = False + framework_found = False for j in policy_groups: if i == os.path.basename(j): found = True break + else: + for f in frameworks: + if i.startswith("%s_" % f): + framework_found = True + break + if framework_found: + found = True + break + if not found: t = 'error' s = "unsupported policy_group '%s'" % i @@ -656,7 +701,11 @@ l = None manual_review = False - aa_type = self._get_policy_group_type(vendor, version, i) + if framework_found: + aa_type = 'framework' + else: + aa_type = self._get_policy_group_type(vendor, version, + i) if i == "debug": t = 'error' s = "(REJECT) %s policy group " % aa_type + \ @@ -668,9 +717,11 @@ if i == "debug": l = 'http://askubuntu.com/a/562123/94326' manual_review = True + elif aa_type == 'framework': + s = "OK (matches '%s' framework)" % i.split('_')[0] elif aa_type != "common": t = 'error' - s = "policy group '%s' has" % i + \ + s = "policy group '%s' has " % i + \ "unknown type '%s'" % (aa_type) self._add_result(t, n, s, l, manual_review=manual_review) @@ -720,7 +771,7 @@ (f, m) = self._get_security_manifest(app) t = 'info' - n = 'ignored_fields (%s)' % f + n = 'required_fields (%s)' % f s = "OK" not_found = [] for i in self.required_fields: @@ -751,3 +802,487 @@ "could not find '%s' in profile" % v) continue self._add_result(t, n, s) + + # This will be really nice to get rid of when the click compat manifest + # is gone + def _compare_security_yamls(self, yaml, click_m): + '''Compare two security yamls''' + def find_match(name, key, value, my_dict): + if 'name' in my_dict and my_dict['name'] == name and \ + key in my_dict and my_dict[key] == value: + return True + return False + + for first in [yaml, click_m]: + if first == yaml: + second = click_m + second_m = "click-manifest" + first_m = "package.yaml" + else: + second = yaml + first_m = "click-manifest" + second_m = "package.yaml" + for exe_t in ['binaries', 'services']: + t = 'info' + n = 'yaml_%s' % exe_t + s = 'OK' + if exe_t in first and exe_t not in second: + t = 'error' + s = "%s missing '%s'" % (second_m, exe_t) + elif exe_t not in first and exe_t in second: + t = 'error' + s = "%s has extra '%s'" % (second_m, exe_t) + self._add_result(t, n, s) + + if t == 'error': + continue + elif exe_t not in first and exe_t not in second: + continue + + t = 'info' + n = 'yaml_%s_entries' % exe_t + s = 'OK' + if len(first[exe_t]) < len(second[exe_t]): + t = 'error' + s = "%s has extra '%s' entries" % (second_m, exe_t) + self._add_result(t, n, s) + + for fapp in first[exe_t]: + t = 'info' + n = 'yaml_%s_%s' % (exe_t, fapp['name']) + s = 'OK' + sapp = None + for tmp in second[exe_t]: + if tmp['name'] == fapp['name']: + sapp = tmp + if sapp is None: + t = 'error' + s = "%s missing '%s'" % (second_m, fapp['name']) + self._add_result(t, n, s) + continue + elif first == yaml and "security-override" in fapp or \ + second == yaml and "security-override" in sapp: + # no reason to check security-override since apparmor + # hook entry will point to this file + continue + elif first == yaml and "security-policy" in fapp or \ + second == yaml and "security-policy" in sapp: + # no reason to check security-policy since apparmor + # profile hook is used instead + continue + elif 'caps' not in fapp and 'caps' not in sapp and \ + second == yaml and 'security-template' not in sapp: + # no caps in yaml, policy_groups is empty in click + # manifest, unless security-template is in yaml + t = 'error' + s = "'policy_groups' not found in click manifest " + \ + "(should default to ['networking'])" + self._add_result(t, n, s) + continue + self._add_result(t, n, s) + + for key in ['security-template', 'caps']: + t = 'info' + n = 'yaml_%s_%s' % (exe_t, second_m) + s = 'OK' + + if key not in fapp: + continue + + if key == 'caps': + fapp['caps'] = set(fapp['caps']) + if 'caps' in sapp: + sapp['caps'] = set(sapp['caps']) + + if not find_match(fapp['name'], key, fapp[key], sapp): + # handle snappy defaults for security-template + # and caps + if key == 'security-template' and \ + second == yaml and key not in sapp and \ + key in fapp and fapp[key] == 'default': + # if yaml missing security-template, click may + # specify 'default' + self._add_result(t, n, s) + continue + elif key == 'caps' and second == yaml and \ + 'security-template' in sapp and \ + key not in sapp and \ + (key not in fapp or key in fapp and + fapp[key] == set([])): + # when security-template is specified, then + # caps won't default to 'networking' when + # missing, so click manifest can omit or be [] + self._add_result(t, n, s) + continue + elif key == 'caps' and second == yaml and \ + 'security-template' not in sapp and \ + key not in sapp and key in fapp and \ + fapp[key] == set(['networking']): + # no caps in yaml, policy_groups is networking + # in click manifest wen security-template not + # specified in yaml + self._add_result(t, n, s) + continue + elif key == 'caps' and second == click_m and \ + key not in sapp and key in fapp and \ + fapp[key] == set([]): + # no caps in click manifest, caps is [] in yaml + self._add_result(t, n, s) + continue + t = 'error' + s = "%s has different '%s' for '%s'" % \ + (second_m, key, fapp['name']) + \ + " - '%s:%s' vs '%s:%s'" % (first_m, fapp, + second_m, sapp) + self._add_result(t, n, s) + + def _convert_click_security_to_yaml(self): + '''Convert click manifest to yaml''' + converted = dict() + for app in sorted(self.security_apps): + if 'snappy-systemd' in self.manifest['hooks'][app]: + key = 'services' + elif 'bin-path' in self.manifest['hooks'][app]: + key = 'binaries' + elif app == 'snappy-config': + # the apparmor policy for snappy-config is autogenerated and + # not represented in the yaml + continue + else: + t = 'error' + n = 'yaml_click_%s' % app + s = "click manifest missing 'snappy-systemd/bin-path' for " + \ + "'%s'" % app + self._add_result(t, n, s) + continue + + if key not in converted: + converted[key] = [] + tmp = dict() + tmp['name'] = app + + (f, m) = self._get_security_manifest(app) + if 'template' in m: + tmp['security-template'] = m['template'] + + if 'policy_groups' in m: + tmp['caps'] = m['policy_groups'] + + converted[key].append(copy.deepcopy(tmp)) + + for app in sorted(self.security_apps_profiles): + if 'snappy-systemd' in self.manifest['hooks'][app]: + key = 'services' + elif 'bin-path' in self.manifest['hooks'][app]: + key = 'binaries' + else: + t = 'error' + n = 'yaml_click_%s' % app + s = "click manifest missing 'snappy-systemd/bin-path' for " + \ + "'%s'" % app + self._add_result(t, n, s) + continue + + if key not in converted: + converted[key] = [] + tmp = dict() + tmp['name'] = app + + (f, p) = self._get_security_profile(app) + tmp['security-policy'] = {'apparmor': f} + + converted[key].append(copy.deepcopy(tmp)) + + return copy.deepcopy(converted) + + def check_security_yaml_and_click(self): + '''Verify click and security yaml are in sync (not including + override) + ''' + if not self.is_snap or self.pkg_yaml['type'] in self.sec_skipped_types: + return + + # setup a small dict that is a subset of self.pkg_yaml + yml = dict() + for exe_t in ['binaries', 'services']: + if exe_t not in self.pkg_yaml: + continue + yml[exe_t] = copy.deepcopy(self.pkg_yaml[exe_t]) + for item in yml[exe_t]: + # account for binaries doing different things with 'name/exec' + if exe_t == 'binaries' and 'exec' not in item and \ + '/' in item['name']: + item['name'] = os.path.basename(item['name']) + + converted = self._convert_click_security_to_yaml() + + # don't compare the security yaml and the click if the yaml isn't + # formatted right. This avoids confusing errors for the user + error = False + for exe_t in yml.keys(): + for item in yml[exe_t]: + if 'security-template' in item and \ + not isinstance(item['security-template'], str): + error = True + continue + elif 'caps' in item and not isinstance(item['caps'], list): + error = True + continue + if error: + t = 'info' + n = 'yaml_and_click' + s = "SKIPPED (yaml errors)" + self._add_result(t, n, s) + return + + self._compare_security_yamls(yml, converted) + + def check_security_yaml_override_and_click(self): + '''Verify click and security yaml override are in sync''' + if not self.is_snap or self.pkg_yaml['type'] in self.sec_skipped_types: + return + + for exe_t in ['services', 'binaries']: + if exe_t not in self.pkg_yaml: + continue + + for item in self.pkg_yaml[exe_t]: + if 'name' not in item: + t = 'error' + n = 'yaml_override_click_name' + s = "package.yaml malformed. Could not find 'name' " + \ + "for entry in '%s'" % item + self._add_result(t, n, s) + continue + + app = item['name'] + t = 'info' + n = 'yaml_override_click_%s' % app + s = "OK" + if 'security-override' not in item: + s = "OK (skipping unspecified override)" + elif 'apparmor' not in item['security-override']: + t = 'error' + s = "'apparmor' not specified in 'security-override' " + \ + "for '%s'" % app + elif item['security-override']['apparmor'] not in \ + self.security_manifests: + t = 'error' + s = "'%s' not found in click manifest for '%s'" % \ + (item['security-override']['apparmor'], app) + # NOTE: we skip 'seccomp' because there isn't currently a + # click hook for it + self._add_result(t, n, s) + + def check_security_yaml_override(self): + '''Verify security yaml override''' + if not self.is_snap: + return + + for exe_t in ['services', 'binaries']: + if exe_t not in self.pkg_yaml: + continue + + for item in self.pkg_yaml[exe_t]: + if 'name' not in item: + t = 'error' + n = 'yaml_override_name' + s = "package.yaml malformed. Could not find 'name' " + \ + "for entry in '%s'" % item + self._add_result(t, n, s) + continue + + app = item['name'] + t = 'info' + n = 'yaml_override_format_%s' % app + s = "OK" + if 'security-override' not in item: + s = "OK (skipping unspecified override)" + elif 'apparmor' not in item['security-override']: + t = 'error' + s = "'apparmor' not specified in 'security-override' " + \ + "for '%s'" % app + elif 'seccomp' not in item['security-override']: + t = 'error' + s = "'seccomp' not specified in 'security-override' " + \ + "for '%s'" % app + self._add_result(t, n, s) + + def check_security_yaml_policy(self): + '''Verify security yaml policy''' + if not self.is_snap: + return + + for exe_t in ['services', 'binaries']: + if exe_t not in self.pkg_yaml: + continue + + for item in self.pkg_yaml[exe_t]: + if 'name' not in item: + t = 'error' + n = 'yaml_policy_name' + s = "package.yaml malformed. Could not find 'name' " + \ + "for entry in '%s'" % item + self._add_result(t, n, s) + continue + + app = item['name'] + t = 'info' + n = 'yaml_policy_format_%s' % app + s = "OK" + if 'security-policy' not in item: + s = "OK (skipping unspecified policy)" + elif 'apparmor' not in item['security-policy']: + t = 'error' + s = "'apparmor' not specified in 'security-policy' " + \ + "for '%s'" % app + elif 'seccomp' not in item['security-policy']: + t = 'error' + s = "'seccomp' not specified in 'security-policy' for " + \ + "'%s'" % app + self._add_result(t, n, s) + + # TODO: error if specified + + def check_security_yaml_combinations(self): + '''Verify security yaml uses valid combinations''' + if not self.is_snap or self.pkg_yaml['type'] in self.sec_skipped_types: + return + + for exe_t in ['services', 'binaries']: + if exe_t not in self.pkg_yaml: + continue + for item in self.pkg_yaml[exe_t]: + if 'name' not in item: + t = 'error' + n = 'yaml_combinations_name' + s = "package.yaml malformed. Could not find 'name' " + \ + "for entry in '%s'" % item + self._add_result(t, n, s) + continue + + app = item['name'] + + t = 'info' + n = 'yaml_combinations_%s' % app + s = "OK" + if "security-policy" in item: + for i in ['security-override', 'security-template', + 'caps']: + if i in item: + t = 'error' + s = "Found '%s' with 'security-policy'" % (i) + break + elif "security-override" in item: + for i in ['security-policy', 'security-template', 'caps']: + if i in item: + t = 'error' + s = "Found '%s' with 'security-override'" % (i) + break + self._add_result(t, n, s) + + def check_security_template(self): + '''Check snap security-template''' + if not self.is_snap or self.pkg_yaml['type'] in self.sec_skipped_types: + return + + for exe_t in ['services', 'binaries']: + if exe_t not in self.pkg_yaml: + continue + for item in self.pkg_yaml[exe_t]: + if 'security-template' not in item: + tmpl = "" + else: + tmpl = item['security-template'] + + if 'name' not in item: + t = 'error' + n = 'yaml_security-template_name' + s = "package.yaml malformed. Could not find 'name' " + \ + "for entry in '%s'" % item + self._add_result(t, n, s) + continue + + # Handle bin/exec concept with bianries + app = os.path.basename(item['name']) + + t = 'info' + n = 'yaml_security-template_%s' % app + s = "OK" + if not isinstance(tmpl, str): + t = 'error' + s = "'%s/%s' malformed: '%s' is not str" % (exe_t, app, + tmpl) + self._add_result(t, n, s) + continue + self._add_result(t, n, s) + + t = 'info' + n = 'yaml_security-template_in_manifest_%s' % app + s = "OK" + if app not in self.manifest['hooks']: + t = 'error' + s = "'%s' not found in click manifest" % app + self._add_result(t, n, s) + continue + elif 'apparmor' not in self.manifest['hooks'][app] and \ + 'apparmor-profile' not in self.manifest['hooks'][app]: + t = 'error' + s = "'apparmor' not found in click manifest for '%s'" % app + self._add_result(t, n, s) + continue + + # TODO: error if not 'common' or is 'unconfined' + + def check_security_caps(self): + '''Check snap caps''' + if not self.is_snap or self.pkg_yaml['type'] in self.sec_skipped_types: + return + + for exe_t in ['services', 'binaries']: + if exe_t not in self.pkg_yaml: + continue + for item in self.pkg_yaml[exe_t]: + if 'caps' not in item: + tmpl = [] + else: + tmpl = item['caps'] + + if 'name' not in item: + t = 'error' + n = 'yaml_caps_name' + s = "package.yaml malformed. Could not find 'name' " + \ + "for entry in '%s'" % item + self._add_result(t, n, s) + continue + + # Handle bin/exec concept with bianries + app = os.path.basename(item['name']) + + t = 'info' + n = 'yaml_caps_%s' % app + s = "OK" + if not isinstance(tmpl, list): + t = 'error' + s = "'%s/%s' malformed: '%s' is not list" % (exe_t, app, + tmpl) + self._add_result(t, n, s) + continue + self._add_result(t, n, s) + + t = 'info' + n = 'yaml_caps_in_manifest_%s' % app + s = "OK" + if app not in self.manifest['hooks']: + t = 'error' + s = "'%s' not found in click manifest" % app + self._add_result(t, n, s) + continue + elif 'apparmor' not in self.manifest['hooks'][app] and \ + 'apparmor-profile' not in self.manifest['hooks'][app]: + t = 'error' + s = "'apparmor' not found in click manifest for '%s'" % app + self._add_result(t, n, s) + continue + + # TODO: error if not 'common' diff -Nru click-reviewers-tools-0.27/clickreviews/cr_systemd.py click-reviewers-tools-0.28/clickreviews/cr_systemd.py --- click-reviewers-tools-0.27/clickreviews/cr_systemd.py 2015-05-04 10:49:48.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_systemd.py 2015-05-27 22:22:55.000000000 +0000 @@ -40,15 +40,24 @@ # - stop # - poststop # - stop-timeout - # - TODO: caps + # - caps (checked in in cr_security.py) + # - security-template (checked in in cr_security.py) + # - security-override (checked in in cr_security.py) + # - security-policy (checked in in cr_security.py) self.required_keys = ['start', 'description'] - self.optional_keys = ['stop', 'poststop', 'stop-timeout'] + \ - self.snappy_exe_security + self.optional_keys = ['stop', + 'poststop', + 'stop-timeout', + 'bus-name', + 'ports' + ] + self.snappy_exe_security self.systemd_files = dict() # click-show-files and tests self.systemd = dict() if self.is_snap and 'services' in self.pkg_yaml: + if len(self.pkg_yaml['services']) == 0: + error("package.yaml malformed: 'services' is empty") for service in self.pkg_yaml['services']: if 'name' not in service: error("package.yaml malformed: required 'name' not found " @@ -156,6 +165,10 @@ (my_dict[app][o], o) else: found = True + elif o == 'ports': + if not isinstance(my_dict[app][o], dict): + t = 'error' + s = "'%s' is not dictionary" % o elif not isinstance(my_dict[app][o], str): t = 'error' s = "'%s' is not a string" % o @@ -338,13 +351,193 @@ self._add_result(t, n, s) def check_service_stop_timeout(self): - '''Check snappy-systemd''' + '''Check snappy-systemd stop-timeout''' self._verify_service_stop_timeout(self.systemd, 'hook') def check_snappy_service_stop_timeout(self): - '''Check snappy package.yaml top-timeout''' + '''Check snappy package.yaml stop-timeout''' if not self.is_snap or 'services' not in self.pkg_yaml: return self._verify_service_stop_timeout(self._create_dict( self.pkg_yaml['services']), 'package_yaml') + + def _verify_service_bus_name(self, pkgname, my_dict, test_str): + for app in sorted(my_dict): + if 'bus-name' not in my_dict[app]: + continue + f = os.path.basename(self.systemd_files[app]) + + t = 'info' + n = '%s_bus-name_empty_%s' % (test_str, f) + s = 'OK' + if len(my_dict[app]['bus-name']) == 0: + t = 'error' + s = "'bus-name' is empty" + self._add_result(t, n, s) + continue + self._add_result(t, n, s) + + t = 'info' + n = '%s_bus-name_format_%s' % (test_str, f) + l = None + s = 'OK' + if not re.search(r'^[A-Za-z0-9][A-Za-z0-9_-]*(\.[A-Za-z0-9][A-Za-z0-9_-]*)+$', + my_dict[app]['bus-name']): + t = 'error' + l = 'http://dbus.freedesktop.org/doc/dbus-specification.html' + s = "'%s' is not of form '^[A-Za-z0-9][A-Za-z0-9_-]*(\\.[A-Za-z0-9][A-Za-z0-9_-]*)+$'" % \ + (my_dict[app]['bus-name']) + self._add_result(t, n, s, l) + + t = 'info' + n = '%s_bus-name_matches_name_%s' % (test_str, f) + s = 'OK' + suggested = [pkgname, + "%s.%s" % (pkgname, app) + ] + if self.is_snap and 'vendor' in self.pkg_yaml: + tmp = self.pkg_yaml['vendor'].split('@') + if len(tmp) > 1: + rev = tmp[1].rstrip('>').split('.') + rev.reverse() + suggested.append("%s.%s" % (".".join(rev), + pkgname)) + suggested.append("%s.%s.%s" % (".".join(rev), + pkgname, + app)) + found = False + for name in suggested: + if my_dict[app]['bus-name'].endswith(name): + found = True + break + if not found: + t = 'error' + s = "'%s' doesn't end with one of: %s" % \ + (my_dict[app]['bus-name'], ", ".join(suggested)) + self._add_result(t, n, s) + + def check_service_bus_name(self): + '''Check snappy-systemd bus-name''' + self._verify_service_bus_name(self.click_pkgname, self.systemd, 'hook') + + def check_snappy_service_bus_name(self): + '''Check snappy package.yaml bus-name''' + if not self.is_snap or 'services' not in self.pkg_yaml: + return + self._verify_service_bus_name(self.pkg_yaml['name'], + self._create_dict( + self.pkg_yaml['services']), + 'package_yaml') + + def _verify_service_ports(self, pkgname, my_dict, test_str): + for app in sorted(my_dict): + if 'ports' not in my_dict[app]: + continue + f = os.path.basename(self.systemd_files[app]) + + t = 'info' + n = '%s_ports_empty_%s' % (test_str, f) + s = 'OK' + if len(my_dict[app]['ports'].keys()) == 0: + t = 'error' + s = "'ports' must contain 'internal' and/or 'external'" + self._add_result(t, n, s) + continue + self._add_result(t, n, s) + + t = 'info' + n = '%s_ports_bad_key_%s' % (test_str, f) + s = 'OK' + badkeys = [] + for i in my_dict[app]['ports'].keys(): + if i not in ['internal', 'external']: + badkeys.append(i) + if len(badkeys) > 0: + t = 'error' + s = "Unknown '%s' found in 'ports'" % ",".join(badkeys) + self._add_result(t, n, s) + + port_pat = re.compile(r'^[0-9]+/[a-z0-9\-]+$') + for key in ['internal', 'external']: + if key not in my_dict[app]['ports']: + continue + + if len(my_dict[app]['ports'][key].keys()) < 1: + t = 'error' + n = '%s_ports_%s_%s' % (test_str, key, f) + s = 'Could not find any %s ports' % key + self._add_result(t, n, s) + continue + + for tagname in my_dict[app]['ports'][key]: + entry = my_dict[app]['ports'][key][tagname] + if len(entry.keys()) < 1: + t = 'error' + n = '%s_ports_%s_%s' % (test_str, key, f) + s = 'Could not find any subkeys for %s' % tagname + self._add_result(t, n, s) + continue + # Annoyingly, the snappy-systemd file uses 'Port' and + # 'Negotiable' instead of 'port' and 'negotiable' from the + # yaml + if (test_str == 'package_yaml' and + 'negotiable' not in entry and + 'port' not in entry) or \ + (test_str == 'hook' and + 'Negotiable' not in entry and + 'Port' not in entry): + t = 'error' + n = '%s_ports_%s_invalid_%s' % (test_str, key, f) + s = "Must specify specify at least 'port' or " + \ + "'negotiable'" + self._add_result(t, n, s) + continue + + # port + subkey = 'port' + if test_str == 'hook': + subkey = 'Port' + t = 'info' + n = '%s_ports_%s_%s_format' % (test_str, tagname, subkey) + s = 'OK' + if subkey not in entry: + s = 'OK (skipped, not found)' + else: + tmp = entry[subkey].split('/') + if not port_pat.search(entry[subkey]) or \ + int(tmp[0]) < 1 or int(tmp[0]) > 65535: + t = 'error' + s = "'%s' should be of form " % entry[subkey] + \ + "'port/protocol' where port is an integer " + \ + "(1-65535) and protocol is found in " + \ + "/etc/protocols" + self._add_result(t, n, s) + + # negotiable + subkey = 'negotiable' + if test_str == 'hook': + subkey = 'Negotiable' + t = 'info' + n = '%s_ports_%s_%s_format' % (test_str, tagname, subkey) + s = 'OK' + if subkey not in entry: + s = 'OK (skipped, not found)' + elif entry[subkey] not in [True, False]: + t = 'error' + s = "'%s: %s' should be either 'yes' or 'no'" % \ + (subkey, entry[subkey]) + self._add_result(t, n, s) + + def check_service_ports(self): + '''Check snappy-systemd ports''' + self._verify_service_ports(self.click_pkgname, self.systemd, 'hook') + + def check_snappy_service_ports(self): + '''Check snappy package.yaml ports''' + if not self.is_snap or 'services' not in self.pkg_yaml: + return + self._verify_service_ports(self.pkg_yaml['name'], + self._create_dict( + self.pkg_yaml['services']), + 'package_yaml') diff -Nru click-reviewers-tools-0.27/clickreviews/cr_tests.py click-reviewers-tools-0.28/clickreviews/cr_tests.py --- click-reviewers-tools-0.27/clickreviews/cr_tests.py 2015-04-30 20:32:50.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/cr_tests.py 2015-06-01 18:35:05.000000000 +0000 @@ -31,6 +31,7 @@ TEST_CONTROL = "" TEST_MANIFEST = "" TEST_PKG_YAML = "" +TEST_HASHES_YAML = "" TEST_README_MD = "" TEST_SECURITY = dict() TEST_SECURITY_PROFILES = dict() @@ -74,11 +75,41 @@ return io.StringIO(TEST_PKG_YAML) +def _extract_hashes_yaml(self): + '''Pretend we read the hashes.yaml file''' + return io.StringIO(TEST_HASHES_YAML) + + +def _path_join(self, d, fn): + '''Pretend we have a tempdir''' + return os.path.join("/fake", fn) + + +def _get_sha512sum(self, fn): + '''Pretend we found performed a sha512''' + (rc, out) = cr_common.cmd(['sha512sum', os.path.realpath(__file__)]) + if rc != 0: + return None + return out.split()[0] + + +def _extract_statinfo(self, fn): + '''Pretend we found performed an os.stat()''' + return os.stat(os.path.realpath(__file__)) + + def _extract_readme_md(self): '''Pretend we read the meta/readme.md file''' return TEST_README_MD +def _check_innerpath_executable(self, fn): + '''Pretend we a file''' + if '.nonexec' in fn: + return False + return True + + def _extract_click_frameworks(self): '''Pretend we enumerated the click frameworks''' return ["ubuntu-sdk-13.10", @@ -240,9 +271,22 @@ 'clickreviews.cr_common.ClickReview._extract_package_yaml', _extract_package_yaml)) patches.append(patch( + 'clickreviews.cr_common.ClickReview._extract_hashes_yaml', + _extract_hashes_yaml)) +patches.append(patch( + 'clickreviews.cr_common.ClickReview._path_join', + _path_join)) +patches.append(patch( + 'clickreviews.cr_common.ClickReview._get_sha512sum', + _get_sha512sum)) +patches.append(patch( + 'clickreviews.cr_common.ClickReview._extract_statinfo', + _extract_statinfo)) +patches.append(patch( 'clickreviews.cr_common.ClickReview._extract_click_frameworks', _extract_click_frameworks)) patches.append(patch('clickreviews.cr_common.unpack_click', _mock_func)) +patches.append(patch('clickreviews.cr_common.raw_unpack_pkg', _mock_func)) patches.append(patch('clickreviews.cr_common.ClickReview._list_all_files', _mock_func)) patches.append(patch( @@ -261,6 +305,9 @@ patches.append(patch( 'clickreviews.cr_lint.ClickReviewLint._extract_readme_md', _extract_readme_md)) +patches.append(patch( + 'clickreviews.cr_lint.ClickReviewLint._check_innerpath_executable', + _check_innerpath_executable)) # security overrides patches.append(patch( @@ -400,6 +447,9 @@ self.test_control['Architecture']) self._update_test_pkg_yaml() + self.test_hashes_yaml = dict() + self._update_test_hashes_yaml() + self.test_readme_md = self.test_control['Description'] self._update_test_readme_md() @@ -514,6 +564,12 @@ default_flow_style=False, indent=4) + def _update_test_hashes_yaml(self): + global TEST_HASHES_YAML + TEST_HASHES_YAML = yaml.dump(self.test_hashes_yaml, + default_flow_style=False, + indent=4) + def _update_test_readme_md(self): global TEST_README_MD TEST_README_MD = self.test_readme_md @@ -736,6 +792,11 @@ self.test_pkg_yaml[key] = value self._update_test_pkg_yaml() + def set_test_hashes_yaml(self, yaml): + '''Set hashes.yaml to yaml''' + self.test_hashes_yaml = yaml + self._update_test_hashes_yaml() + def set_test_readme_md(self, contents): '''Set contents of meta/readme.md''' if contents is None: @@ -745,12 +806,57 @@ self._update_test_readme_md() def set_test_security_manifest(self, app, key, value): - '''Set key in security manifest to value. If value is None, remove - key''' + '''Set key in security manifest and package.yaml to value. If value is + None, remove key, if key is None, remove app. + ''' + # package.yaml - we don't know if it is a service or a binary with + # these values, so just use 'binaries' + if key is None: + if 'binaries' in self.test_pkg_yaml: + for b in self.test_pkg_yaml['binaries']: + if 'name' in b and b['name'] == app: + self.test_pkg_yaml['binaries'].remove(b) + break + elif value is None: + if 'binaries' in self.test_pkg_yaml: + found = False + for b in self.test_pkg_yaml['binaries']: + if 'name' in b and b['name'] == app: + if key in b: + b.remove(key) + found = True + break + else: + found = False + k = key + if key == 'template': + k = 'security-template' + elif key == 'policy_groups': + k = 'caps' + if 'binaries' in self.test_pkg_yaml: + for b in self.test_pkg_yaml['binaries']: + if 'name' in b and b['name'] == app: + # Found the entry, so update key/value + b[k] = value + found = True + break + # Did not find the entry, so create one + if not found: + if 'binaries' not in self.test_pkg_yaml: + self.test_pkg_yaml['binaries'] = [] + self.test_pkg_yaml['binaries'].append({'name': app, + k: value}) + self._update_test_pkg_yaml() + + # click manifest if app not in self.test_security_manifests: self.test_security_manifests[app] = dict() - if value is None: + if key is None: + if app in self.test_security_manifests: + del self.test_security_manifests[app] + del self.test_manifest["hooks"][app] + elif value is None: if key in self.test_security_manifests[app]: self.test_security_manifests[app].pop(key, None) else: @@ -973,8 +1079,8 @@ self._update_test_framework_policy_unknown() def set_test_systemd(self, app, key, value): - '''Set systemd entries. If key is None, snappy-systemd from manifest - and yaml. + '''Set systemd entries. If key is None, remove snappy-systemd from + manifest and yaml. Note the click manifest and the package.yaml use different storage types. pkg_yaml['services'] is a list of dictionaries where @@ -1041,6 +1147,8 @@ TEST_MANIFEST = "" global TEST_PKG_YAML TEST_PKG_YAML = "" + global TEST_HASHES_YAML + TEST_HASHES_YAML = "" global TEST_README_MD TEST_README_MD = "" global TEST_SECURITY diff -Nru click-reviewers-tools-0.27/clickreviews/tests/test_cr_desktop.py click-reviewers-tools-0.28/clickreviews/tests/test_cr_desktop.py --- click-reviewers-tools-0.27/clickreviews/tests/test_cr_desktop.py 2015-04-30 20:32:50.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/tests/test_cr_desktop.py 2015-05-22 15:09:05.000000000 +0000 @@ -315,6 +315,32 @@ expected_counts = {'info': 2, 'warn': 0, 'error': 0} self.check_results(r, expected_counts) + def test_check_desktop_exec_webapp_container_html5_launcher_1410(self): + '''Test check_desktop_exec_webapp_container - html5 launcher 14.10''' + self.set_test_manifest("framework", "ubuntu-sdk-14.10") + ex = "ubuntu-html5-app-launcher $@ --www=www" + c = ClickReviewDesktop(self.test_name) + self.set_test_desktop(self.default_appname, + "Exec", + ex) + c.check_desktop_exec_webapp_container() + r = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_desktop_exec_webapp_container_html5_launcher_1510(self): + '''Test check_desktop_exec_webapp_container - html5 launcher 15.10''' + self.set_test_manifest("framework", "ubuntu-sdk-15.10") + ex = "ubuntu-html5-app-launcher $@ --www=www" + c = ClickReviewDesktop(self.test_name) + self.set_test_desktop(self.default_appname, + "Exec", + ex) + c.check_desktop_exec_webapp_container() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + def test_check_desktop_exec_webapp_container_missing_exec(self): '''Test check_desktop_exec_webapp_container - missing exec''' c = ClickReviewDesktop(self.test_name) diff -Nru click-reviewers-tools-0.27/clickreviews/tests/test_cr_lint.py click-reviewers-tools-0.28/clickreviews/tests/test_cr_lint.py --- click-reviewers-tools-0.27/clickreviews/tests/test_cr_lint.py 2015-05-04 10:49:48.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/tests/test_cr_lint.py 2015-06-09 13:07:06.000000000 +0000 @@ -21,6 +21,9 @@ from clickreviews.frameworks import FRAMEWORKS_DATA_URL, USER_DATA_FILE import clickreviews.cr_tests as cr_tests +import os +import stat + class TestClickReviewLint(cr_tests.TestClickReview): """Tests for the lint review tool.""" @@ -30,6 +33,30 @@ cr_tests.mock_patch() super() + def _create_hashes_yaml(self): + # find cr_tests.py since that is what _get_statinfo() is mocked to + # look at. + f = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "../cr_tests.py") + statinfo = os.stat(f) + self.sha512 = cr_tests._get_sha512sum(self, f) + hashes = {'archive-sha512': self.sha512, + 'files': [{'name': 'bin', + 'mode': 'drwxrwxr-x'}, + {'name': 'bin/foo', + 'size': statinfo.st_size, + 'mode': 'f%s' % + stat.filemode(statinfo.st_mode)[1:], + 'sha512': self.sha512}, + {'name': 'barlink', + 'mode': 'lrwxrwxrwx'}, + ] + } + self._test_pkg_files = [] + for i in hashes['files']: + self._test_pkg_files.append(i['name']) + return hashes + def patch_frameworks(self): def _mock_frameworks(self, overrides=None): self.FRAMEWORKS = { @@ -481,14 +508,25 @@ self.check_results(r, expected_counts) def test_check_maintainer_missing(self): - '''Test check_maintainer() - missing''' + '''Test check_maintainer() - missing (click)''' self.set_test_manifest("maintainer", None) c = ClickReviewLint(self.test_name) + c.is_snap = False c.check_maintainer() r = c.click_report expected_counts = {'info': 0, 'warn': 0, 'error': 1} self.check_results(r, expected_counts) + def test_check_maintainer_missing_snap(self): + '''Test check_maintainer() - missing (snap)''' + self.set_test_manifest("maintainer", None) + c = ClickReviewLint(self.test_name) + c.is_snap = True + c.check_maintainer() + r = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + def test_check_maintainer_badformat(self): '''Test check_maintainer() - badly formatted''' self.set_test_manifest("maintainer", "$%^@*") @@ -883,31 +921,64 @@ self.check_results(r, expected_counts) self.check_manual_review(r, 'lint_hooks_redflag_test-app') - def test_snappy_name1(self): + def test_pkgname_toplevel(self): + '''Test check_pkgname - toplevel''' + self.set_test_manifest("name", "foo") + c = ClickReviewLint(self.test_name) + c.is_snap = False + c.check_pkgname() + r = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_pkgname_flat(self): + '''Test check_pkgname - flat''' + self.set_test_manifest("name", "foo.bar") + c = ClickReviewLint(self.test_name) + c.is_snap = False + c.check_pkgname() + r = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_pkgname_reverse_domain(self): + '''Test check_pkgname - reverse domain''' + self.set_test_manifest("name", "com.ubuntu.develeper.baz.foo") + c = ClickReviewLint(self.test_name) + c.is_snap = False + c.check_pkgname() + r = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_snappy_name_toplevel(self): '''Test check_snappy_name - toplevel''' self.set_test_pkg_yaml("name", "foo") c = ClickReviewLint(self.test_name) + c.is_snap = True c.check_snappy_name() r = c.click_report expected_counts = {'info': 1, 'warn': 0, 'error': 0} self.check_results(r, expected_counts) - def test_snappy_name2(self): - '''Test check_snappy_name - flat''' + def test_snappy_name_flat(self): + '''Test check_snappy_name - obsoleted flat''' self.set_test_pkg_yaml("name", "foo.bar") c = ClickReviewLint(self.test_name) + c.is_snap = True c.check_snappy_name() r = c.click_report - expected_counts = {'info': 1, 'warn': 0, 'error': 0} + expected_counts = {'info': None, 'warn': 0, 'error': 1} self.check_results(r, expected_counts) - def test_snappy_name3(self): - '''Test check_snappy_name - reverse domain''' + def test_snappy_name_reverse_domain(self): + '''Test check_snappy_name - obsoleted reverse domain''' self.set_test_pkg_yaml("name", "com.ubuntu.develeper.baz.foo") c = ClickReviewLint(self.test_name) + c.is_snap = True c.check_snappy_name() r = c.click_report - expected_counts = {'info': 1, 'warn': 0, 'error': 0} + expected_counts = {'info': None, 'warn': 0, 'error': 1} self.check_results(r, expected_counts) def test_snappy_name_bad(self): @@ -1357,3 +1428,366 @@ r = c.click_report expected_counts = {'info': None, 'warn': 0, 'error': 1} self.check_results(r, expected_counts) + + def test_check_snappy_services_and_binaries1(self): + '''Test check_snappy_services_and_binaries() - different''' + self.set_test_pkg_yaml("name", self.test_name.split('_')[0]) + self.set_test_pkg_yaml("services", [{"name": "foo", + "start": "bin/foo"}]) + self.set_test_pkg_yaml("binaries", [{"name": "bar"}]) + c = ClickReviewLint(self.test_name) + c.check_snappy_services_and_binaries() + r = c.click_report + expected_counts = {'info': 2, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_services_and_binaries2(self): + '''Test check_snappy_services_and_binaries() - different (exec)''' + self.set_test_pkg_yaml("name", self.test_name.split('_')[0]) + self.set_test_pkg_yaml("services", [{"name": "foo", + "start": "bin/foo"}]) + self.set_test_pkg_yaml("binaries", [{"name": "bar", + "exec": "bin/foo"}]) + c = ClickReviewLint(self.test_name) + c.check_snappy_services_and_binaries() + r = c.click_report + expected_counts = {'info': 2, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_services_and_binaries3(self): + '''Test check_snappy_services_and_binaries() - same''' + self.set_test_pkg_yaml("name", self.test_name.split('_')[0]) + self.set_test_pkg_yaml("services", [{"name": "foo", + "start": "bin/foo"}]) + self.set_test_pkg_yaml("binaries", [{"name": "foo"}]) + c = ClickReviewLint(self.test_name) + c.check_snappy_services_and_binaries() + r = c.click_report + expected_counts = {'info': 0, 'warn': 0, 'error': 2} + self.check_results(r, expected_counts) + + m = r['error']['lint_snappy_foo_in_services']['text'] + self.assertIn("'foo' in both 'services' and 'binaries'", m) + m = r['error']['lint_snappy_foo_in_binaries']['text'] + self.assertIn("'foo' in both 'services' and 'binaries'", m) + + def test_check_snappy_services_and_binaries4(self): + '''Test check_snappy_services_and_binaries() - same (subdir)''' + self.set_test_pkg_yaml("name", self.test_name.split('_')[0]) + self.set_test_pkg_yaml("services", [{"name": "foo", + "start": "bin/foo"}]) + self.set_test_pkg_yaml("binaries", [{"name": "bin/foo"}]) + c = ClickReviewLint(self.test_name) + c.check_snappy_services_and_binaries() + r = c.click_report + expected_counts = {'info': 0, 'warn': 0, 'error': 2} + self.check_results(r, expected_counts) + + m = r['error']['lint_snappy_foo_in_services']['text'] + self.assertIn("'foo' in both 'services' and 'binaries'", m) + m = r['error']['lint_snappy_foo_in_binaries']['text'] + self.assertIn("'foo' in both 'services' and 'binaries'", m) + + def test_check_snappy_services_and_binaries5(self): + '''Test check_snappy_services_and_binaries() - same (exec, subdir)''' + self.set_test_pkg_yaml("name", self.test_name.split('_')[0]) + self.set_test_pkg_yaml("services", [{"name": "foo", + "start": "bin/foo"}]) + self.set_test_pkg_yaml("binaries", [{"name": "foo", + "exec": "bin/foo"}]) + c = ClickReviewLint(self.test_name) + c.check_snappy_services_and_binaries() + r = c.click_report + expected_counts = {'info': 0, 'warn': 0, 'error': 2} + self.check_results(r, expected_counts) + + m = r['error']['lint_snappy_foo_in_services']['text'] + self.assertIn("'foo' in both 'services' and 'binaries'", m) + m = r['error']['lint_snappy_foo_in_binaries']['text'] + self.assertIn("'foo' in both 'services' and 'binaries'", m) + + def test_check_snappy_hashes_click(self): + '''Test check_snappy_hashes() - click''' + c = ClickReviewLint(self.test_name) + c.is_snap = False + c.check_snappy_hashes() + r = c.click_report + # clicks don't have hashes.yaml, so should have no output + expected_counts = {'info': 0, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_archive_sha512_missing(self): + '''Test check_snappy_hashes() - archive-sha512 missing''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': 0, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_archive_sha512_invalid(self): + '''Test check_snappy_hashes() - archive-sha512 invalid''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + yaml['archive-sha512'] = 'deadbeef' + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': 0, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_hashes_archive-sha512_valid']['text'] + self.assertIn("hash mismatch: 'deadbeef' != '%s'" % self.sha512, m) + + def test_check_snappy_hashes_archive_files_missing(self): + '''Test check_snappy_hashes() - files missing''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + del yaml['files'] + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_hashes_files_present']['text'] + self.assertIn("'files' not found in hashes.yaml", m) + + def test_check_snappy_hashes_archive_files_ok(self): + '''Test check_snappy_hashes() - ok''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + c.pkg_files = self._test_pkg_files + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': 4, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_archive_files_missing_name(self): + '''Test check_snappy_hashes() - missing name''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + del yaml['files'][0]['name'] + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_archive_files_missing_mode(self): + '''Test check_snappy_hashes() - missing mode''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + del yaml['files'][0]['mode'] + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_archive_files_malformed_mode(self): + '''Test check_snappy_hashes() - malformed mode''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + yaml['files'][0]['mode'] += 'extra' + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_archive_files_bad_mode_entry(self): + '''Test check_snappy_hashes() - bad mode entry''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + count = 0 + orig_mode = None + for e in yaml['files']: + s = list(yaml['files'][count]['mode']) + if e['name'] == 'bin/foo': + # keep track of the other parts of the on disk mode + orig_mode = s + orig_mode[3] = 'S' + s[3] = 'S' + yaml['files'][count]['mode'] = "".join(s) + count += 1 + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_file_mode']['text'] + self.assertIn("found errors in hashes.yaml: unusual mode '%s' for entry 'bin/foo'" % "".join(orig_mode), m) + + def test_check_snappy_hashes_archive_files_mode_world_write(self): + '''Test check_snappy_hashes() - mode world write''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + count = 0 + for e in yaml['files']: + s = list(e['mode']) + if e['name'] == 'bin/foo' or e['name'] == 'bin': + s[-2] = 'w' + s[-5] = 'w' + yaml['files'][count]['mode'] = "".join(s) + count += 1 + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_file_mode']['text'] + self.assertIn("found errors in hashes.yaml: 'bin' is world-writable, mode 'frw-rw-rw-' for 'bin/foo' is world-writable", m) + + def test_check_snappy_hashes_archive_files_mode_mismatch(self): + '''Test check_snappy_hashes() - mode mismatch''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + count = 0 + orig_mode = None + for e in yaml['files']: + if e['mode'].startswith('f'): + # keep track of the other parts of the on disk mode + orig_mode = e['mode'][1:] + yaml['files'][count]['mode'] = "f---------" + break + count += 1 + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_file_mode']['text'] + self.assertIn("found errors in hashes.yaml: mode '---------' != '%s' for 'bin/foo'" % orig_mode, m) + + def test_check_snappy_hashes_archive_files_mode_bad_symlink(self): + '''Test check_snappy_hashes() - mode bad symlink''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + yaml['files'].append({'name': 'badlink', 'mode': 'lrwxrwxr-x'}) + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_file_mode']['text'] + self.assertIn("found errors in hashes.yaml: unusual mode 'lrwxrwxr-x' for entry 'badlink'", m) + + def test_check_snappy_hashes_archive_files_mode_devices(self): + '''Test check_snappy_hashes() - mode devices''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + yaml['files'].append({'name': 'badblock', 'mode': 'brw-rw-r--'}) + yaml['files'].append({'name': 'badchar', 'mode': 'crw-rw-r--'}) + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_file_mode']['text'] + self.assertIn("found errors in hashes.yaml: illegal file mode 'b': 'brw-rw-r--' for 'badblock', illegal file mode 'c': 'crw-rw-r--' for 'badchar'", m) + + def test_check_snappy_hashes_archive_files_missing_size(self): + '''Test check_snappy_hashes() - missing size''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + count = 0 + for e in yaml['files']: + if e['mode'].startswith('f'): + del yaml['files'][count]['size'] + break + count += 1 + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_archive_files_invalid_size(self): + '''Test check_snappy_hashes() - invalid size''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + count = 0 + for e in yaml['files']: + if e['mode'].startswith('f'): + orig_size = e['size'] + new_size = orig_size + 1 + yaml['files'][count]['size'] = new_size + break + count += 1 + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_file_mode']['text'] + self.assertIn("found errors in hashes.yaml: size " + + "%d != %d for 'bin/foo'" % (new_size, orig_size), m) + + def test_check_snappy_hashes_archive_files_missing_sha512(self): + '''Test check_snappy_hashes() - missing sha512''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + count = 0 + for e in yaml['files']: + if e['mode'].startswith('f'): + del yaml['files'][count]['sha512'] + break + count += 1 + self.set_test_hashes_yaml(yaml) + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_hashes_extra(self): + '''Test check_snappy_hashes() - extra''' + c = ClickReviewLint(self.test_name) + c.is_snap = True + yaml = self._create_hashes_yaml() + self.set_test_hashes_yaml(yaml) + c.pkg_files = self._test_pkg_files + c.pkg_files.append("extrafile") + c.check_snappy_hashes() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + m = r['error']['lint_hashes_extra_files']['text'] + self.assertIn("found extra files not listed in hashes.yaml: extrafile", + m) + + def test_snappy_config(self): + '''Test check_snappy_config()''' + c = ClickReviewLint(self.test_name) + c.unpack_dir = "/nonexistent" + c.pkg_files.append(os.path.join(c.unpack_dir, 'meta/hooks/config')) + c.is_snap = True + c.check_snappy_config() + r = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_snappy_config_nonexecutable(self): + '''Test check_snappy_config() - not executable''' + c = ClickReviewLint(self.test_name) + c.unpack_dir = "/nonexistent.nonexec" + c.pkg_files.append(os.path.join(c.unpack_dir, + 'meta/hooks/config')) + c.is_snap = True + c.check_snappy_config() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) diff -Nru click-reviewers-tools-0.27/clickreviews/tests/test_cr_security.py click-reviewers-tools-0.28/clickreviews/tests/test_cr_security.py --- click-reviewers-tools-0.27/clickreviews/tests/test_cr_security.py 2015-05-04 10:49:48.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/tests/test_cr_security.py 2015-06-10 21:02:05.000000000 +0000 @@ -32,6 +32,16 @@ self.default_security_json = "%s.apparmor" % \ self.default_appname + def _set_yaml_binary(self, entries, name=None): + d = dict() + if name is None: + d['name'] = self.default_appname + else: + d['name'] = name + for (key, value) in entries: + d[key] = value + self.set_test_pkg_yaml("binaries", [d]) + def test_check_policy_version_vendor(self): '''Test check_policy_version() - valid''' for v in [1.0]: # update when have more vendor policy @@ -482,6 +492,52 @@ expected_counts = {'info': None, 'warn': 0, 'error': 1} self.check_results(report, expected_counts) + def test_check_template_snappy_framework_deprecated(self): + '''Test check_template() - in deprecated framework declaration''' + self.set_test_pkg_yaml("framework", "fwk") + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "template", "fwk_foo") + c.check_template() + report = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_template_snappy_framework_deprecated2(self): + '''Test check_template() - in deprecated framework declaration list''' + self.set_test_pkg_yaml("framework", "fwk, somethingelse") + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "template", "fwk_foo") + c.check_template() + report = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_template_snappy_frameworks(self): + '''Test check_template() - in frameworks declaration''' + self.set_test_pkg_yaml("frameworks", ["fwk"]) + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "template", "fwk_foo") + c.check_template() + report = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_template_snappy_framework_type(self): + '''Test check_template() - type framework''' + self.set_test_pkg_yaml("type", "framework") + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "template", + "%s_foo" % + self.test_name.split('_')[0]) + c.check_template() + report = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + def test_check_template_webapp(self): '''Test check_template() - webapp''' c = ClickReviewSecurity(self.test_name) @@ -762,7 +818,7 @@ c = ClickReviewSecurity(self.test_name) c.check_policy_groups() report = c.click_report - expected_counts = {'info': 0, 'warn': 1, 'error': 0} + expected_counts = {'info': 1, 'warn': 0, 'error': 0} self.check_results(report, expected_counts) def test_check_policy_groups_bad_policy_version(self): @@ -833,6 +889,54 @@ expected_counts = {'info': None, 'warn': 0, 'error': 1} self.check_results(report, expected_counts) + def test_check_policy_groups_snappy_framework_deprecated(self): + '''Test check_policy_groups() - in deprecated framework declaration''' + self.set_test_pkg_yaml("framework", "fwk") + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "policy_groups", ["fwk_foo"]) + c.check_policy_groups() + report = c.click_report + expected_counts = {'info': 4, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_policy_groups_snappy_framework_deprecated2(self): + '''Test check_policy_groups() - in deprecated framework declaration + list + ''' + self.set_test_pkg_yaml("framework", "fwk, somethingelse") + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "policy_groups", ["fwk_foo"]) + c.check_policy_groups() + report = c.click_report + expected_counts = {'info': 4, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_policy_groups_snappy_frameworks(self): + '''Test check_policy_groups() - in frameworks declaration''' + self.set_test_pkg_yaml("frameworks", ["fwk"]) + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "policy_groups", ["fwk_foo"]) + c.check_policy_groups() + report = c.click_report + expected_counts = {'info': 4, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_policy_groups_snappy_framework_type(self): + '''Test check_policy_groups() - type framework''' + self.set_test_pkg_yaml("type", "framework") + c = ClickReviewSecurity(self.test_name) + self.set_test_security_manifest(self.default_appname, + "policy_groups", + ["%s_foo" % + self.test_name.split('_')[0]]) + c.check_policy_groups() + report = c.click_report + expected_counts = {'info': 4, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + def test_check_policy_groups_pushhelper_no_hook(self): '''Test check_policy_groups_pushhelper() - no hook''' c = ClickReviewSecurity(self.test_name) @@ -1154,3 +1258,644 @@ report = c.click_report expected_counts = {'info': None, 'warn': 1, 'error': 0} self.check_results(report, expected_counts) + + def test_check_security_template_default(self): + '''Test check_security_template() - default''' + self.set_test_security_manifest(self.default_appname, "template", None) + self._set_yaml_binary([]) + c = ClickReviewSecurity(self.test_name) + c.check_security_template() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_template_default_with_exec(self): + '''Test check_security_template() - default with exec''' + self.set_test_security_manifest(self.default_appname, "template", None) + self._set_yaml_binary([('exec', 'bin/%s' % self.default_appname)]) + c = ClickReviewSecurity(self.test_name) + c.check_security_template() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_template_nondefault(self): + '''Test check_security_template() - nondefault''' + self.set_test_security_manifest(self.default_appname, + "template", "nondefault") + self._set_yaml_binary([('security-template', 'nondefault')]) + c = ClickReviewSecurity(self.test_name) + c.check_security_template() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_template_bad(self): + '''Test check_security_template() - {}''' + self._set_yaml_binary([('security-template', {})]) + c = ClickReviewSecurity(self.test_name) + c.check_security_template() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_caps_default(self): + '''Test check_security_caps() - default''' + self.set_test_security_manifest(self.default_appname, + "caps", ['networking']) + self._set_yaml_binary([('caps', ['networking'])]) + c = ClickReviewSecurity(self.test_name) + c.check_security_caps() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_caps_default_with_exec(self): + '''Test check_security_caps() - default with exec''' + self.set_test_security_manifest(self.default_appname, + "caps", ['networking']) + self._set_yaml_binary([('exec', 'bin/%s' % self.default_appname), + ('caps', ['networking'])]) + c = ClickReviewSecurity(self.test_name) + c.check_security_caps() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_caps_nondefault(self): + '''Test check_security_caps() - nondefault''' + self.set_test_security_manifest(self.default_appname, + "caps", []) + self._set_yaml_binary([('caps', [])]) + c = ClickReviewSecurity(self.test_name) + c.check_security_caps() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_caps_bad(self): + '''Test check_security_caps() - {}''' + self._set_yaml_binary([('caps', {})]) + c = ClickReviewSecurity(self.test_name) + c.check_security_caps() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_name_relative(self): + '''Test check_security_yaml_and_click() - relative path''' + self._set_yaml_binary([('caps', ['networking'])], + name="bin/%s" % self.default_appname) + c = ClickReviewSecurity(self.test_name) + + # update the manifest and test_manifest + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 5, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_name_exec(self): + '''Test check_security_yaml_and_click() - uses exec''' + self._set_yaml_binary([('caps', ['networking']), + ('exec', "bin/%s" % self.default_appname)], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + + # update the manifest and test_manifest + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 6, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_matching_template(self): + '''Test check_security_yaml_and_click() - matching default template''' + self._set_yaml_binary([('caps', ['networking'])]) + self.set_test_security_manifest(self.default_appname, + "template", "default") + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + del c.pkg_yaml['binaries'][0]['security-template'] + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 6, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_security_override(self): + '''Test check_security_yaml_and_click() - security-override''' + self._set_yaml_binary([('security-template', 'default')]) + self._set_yaml_binary([('security-override', {})]) + self.set_test_security_manifest(self.default_appname, + "template", "default") + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_security_policy(self): + '''Test check_security_yaml_and_click() - security-policy''' + self._set_yaml_binary([('security-template', 'default')]) + self._set_yaml_binary([('security-policy', {})]) + self.set_test_security_manifest(self.default_appname, + "template", "default") + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_matching_caps(self): + '''Test check_security_yaml_and_click() - matching default caps''' + self._set_yaml_binary([('caps', [])]) + self.set_test_security_manifest(self.default_appname, + "policy_groups", ['networking']) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + del c.pkg_yaml['binaries'][0]['caps'] + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 5, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_matching_no_caps(self): + '''Test check_security_yaml_and_click() - matching no caps''' + self._set_yaml_binary([('caps', [])]) + self.set_test_security_manifest(self.default_appname, + "policy_groups", None) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 5, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_matching_no_caps_template(self): + '''Test check_security_yaml_and_click() - matching no caps with + template + ''' + self._set_yaml_binary([('security-template', 'nondefault')]) + self.set_test_security_manifest(self.default_appname, + "policy_groups", None) + self.set_test_security_manifest(self.default_appname, + "template", "nondefault") + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 5, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch0(self): + '''Test check_security_yaml_and_click() - missing app in hooks''' + self._set_yaml_binary([]) + self.set_test_security_manifest(self.default_appname, + "template", None) + c = ClickReviewSecurity(self.test_name) + + del c.manifest["hooks"][self.default_appname] + self._update_test_manifest() + c.security_apps.remove(self.default_appname) + + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch1(self): + '''Test check_security_yaml_and_click() - missing bin-path in hooks''' + self._set_yaml_binary([]) + c = ClickReviewSecurity(self.test_name) + + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 2} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch2(self): + '''Test check_security_yaml_and_click() - missing apparmor in hooks''' + self._set_yaml_binary([]) + c = ClickReviewSecurity(self.test_name) + + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + del c.manifest["hooks"][self.default_appname]['apparmor'] + c.security_apps.remove(self.default_appname) + self._update_test_manifest() + + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch3(self): + '''Test check_security_yaml_and_click() - missing security-template''' + self._set_yaml_binary([('caps', ['networking'])]) + self.set_test_security_manifest(self.default_appname, + "template", "nondefault") + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + del c.pkg_yaml['binaries'][0]['security-template'] + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch4(self): + '''Test check_security_yaml_and_click() - missing click template''' + self._set_yaml_binary([("security-template", "nondefault"), + ('caps', ['networking'])], + name=self.default_appname) + self.set_test_security_manifest(self.default_appname, + "template", None) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch5(self): + '''Test check_security_yaml_and_click() - different templates''' + self.set_test_security_manifest(self.default_appname, + "template", "other") + self._set_yaml_binary([("security-template", "nondefault"), + ('caps', ['networking'])], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 2} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch6(self): + '''Test check_security_yaml_and_click() - missing caps in yaml''' + self._set_yaml_binary([('caps', [])]) + self.set_test_security_manifest(self.default_appname, + "policy_groups", ["1", "2"]) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + del c.pkg_yaml['binaries'][0]['caps'] + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch7(self): + '''Test check_security_yaml_and_click() - missing policy_groups''' + self._set_yaml_binary([('caps', ['networking'])], + name=self.default_appname) + self.set_test_security_manifest(self.default_appname, + "policy_groups", None) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch8(self): + '''Test check_security_yaml_and_click() - different caps/groups''' + self.set_test_security_manifest(self.default_appname, + "policy_groups", ["1", "2"]) + self._set_yaml_binary([('caps', ["3"])], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 2} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch9(self): + '''Test check_security_yaml_and_click() - unordered caps/groups''' + self.set_test_security_manifest(self.default_appname, + "policy_groups", ["1", "2"]) + self._set_yaml_binary([('caps', ["2", "1"])], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 5, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch10(self): + '''Test check_security_yaml_and_click() - missing caps in both''' + self._set_yaml_binary([('caps', [])]) + self.set_test_security_manifest(self.default_appname, + "policy_groups", None) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + del c.pkg_yaml['binaries'][0]['caps'] + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_mismatch11(self): + '''Test check_security_yaml_and_click() - default caps with template''' + self._set_yaml_binary([('security-template', 'nondefault')]) + self.set_test_security_manifest(self.default_appname, + "policy_groups", ['networking']) + self.set_test_security_manifest(self.default_appname, + "template", "nondefault") + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + del c.pkg_yaml['binaries'][0]['caps'] + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_invalid_template(self): + '''Test check_security_yaml_and_click() - invalid template''' + self.set_test_security_manifest(self.default_appname, + "template", "other") + self._set_yaml_binary([("security-template", None), + ('caps', ['networking'])], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_and_click_invalid_caps(self): + '''Test check_security_yaml_and_click() - invalid caps''' + self.set_test_security_manifest(self.default_appname, + "template", "other") + self._set_yaml_binary([("security-template", "nondefault"), + ('caps', None)], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.manifest["hooks"][self.default_appname]['bin-path'] = "bin/path" + c.check_security_yaml_and_click() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations(self): + '''Test check_security_yaml_combinations()''' + self._set_yaml_binary([], name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations1(self): + '''Test check_security_yaml_combinations() - template''' + self._set_yaml_binary([('security-template', 'foo')], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations2(self): + '''Test check_security_yaml_combinations() - caps''' + self._set_yaml_binary([('caps', ['networking'])], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations3(self): + '''Test check_security_yaml_combinations() - template,caps''' + self._set_yaml_binary([('security-template', 'foo'), + ('caps', ['networking'])], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations4(self): + '''Test check_security_yaml_combinations() - override''' + self._set_yaml_binary([('security-override', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations5(self): + '''Test check_security_yaml_combinations() - override, template''' + self._set_yaml_binary([('security-template', 'foo'), + ('security-override', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations6(self): + '''Test check_security_yaml_combinations() - override, caps''' + self._set_yaml_binary([('caps', ['networking']), + ('security-override', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations7(self): + '''Test check_security_yaml_combinations() - override, caps, template + ''' + self._set_yaml_binary([('security-template', 'foo'), + ('caps', ['networking']), + ('security-override', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations8(self): + '''Test check_security_yaml_combinations() - override, caps, template, + policy + ''' + self._set_yaml_binary([('security-template', 'foo'), + ('caps', ['networking']), + ('security-policy', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'}), + ('security-override', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations9(self): + '''Test check_security_yaml_combinations() - policy''' + self._set_yaml_binary([('security-policy', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations10(self): + '''Test check_security_yaml_combinations() - policy, template''' + self._set_yaml_binary([('security-template', 'foo'), + ('security-policy', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations11(self): + '''Test check_security_yaml_combinations() - policy, caps''' + self._set_yaml_binary([('caps', ['networking']), + ('security-policy', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_combinations12(self): + '''Test check_security_yaml_combinations() - policy, caps, template + ''' + self._set_yaml_binary([('security-template', 'foo'), + ('caps', ['networking']), + ('security-policy', {'apparmor': 'foo.aa', + 'seccomp': 'foo.sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_combinations() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_override_and_click(self): + '''Test check_security_yaml_override_and_click()''' + self.set_test_security_manifest(self.default_appname, "template", None) + self._set_yaml_binary([]) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_override_and_click() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_override_and_click_bad(self): + '''Test check_security_yaml_override_and_click() - bad''' + self.set_test_security_manifest(self.default_appname, "template", None) + self._set_yaml_binary([('security-override', {'apparmor': + 'something.else'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_override_and_click() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_override(self): + '''Test check_security_yaml_override()''' + self.set_test_security_manifest(self.default_appname, "template", None) + self._set_yaml_binary([]) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_override() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_override2(self): + '''Test check_security_yaml_override() - seccomp/apparmor specified''' + self._set_yaml_binary([('security-override', {'apparmor': 'aa', + 'seccomp': 'sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_override() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_override_missing1(self): + '''Test check_security_yaml_override() - missing apparmor''' + self._set_yaml_binary([('security-override', {'seccomp': 'sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_override() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_override_missing2(self): + '''Test check_security_yaml_override() - missing seccomp''' + self._set_yaml_binary([('security-override', {'apparmor': 'aa'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_override() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_policy(self): + '''Test check_security_yaml_policy()''' + self.set_test_security_manifest(self.default_appname, "template", None) + self._set_yaml_binary([]) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_policy() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_policy2(self): + '''Test check_security_yaml_policy() - seccomp/apparmor specified''' + self._set_yaml_binary([('security-policy', {'apparmor': 'aa', + 'seccomp': 'sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_policy() + report = c.click_report + expected_counts = {'info': 1, 'warn': 0, 'error': 0} + self.check_results(report, expected_counts) + + def test_check_security_yaml_policy_missing1(self): + '''Test check_security_yaml_policy() - missing apparmor''' + self._set_yaml_binary([('security-policy', {'seccomp': 'sc'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_policy() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) + + def test_check_security_yaml_policy_missing2(self): + '''Test check_security_yaml_policy() - missing seccomp''' + self._set_yaml_binary([('security-policy', {'apparmor': 'aa'})], + name=self.default_appname) + c = ClickReviewSecurity(self.test_name) + c.check_security_yaml_policy() + report = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(report, expected_counts) diff -Nru click-reviewers-tools-0.27/clickreviews/tests/test_cr_systemd.py click-reviewers-tools-0.28/clickreviews/tests/test_cr_systemd.py --- click-reviewers-tools-0.27/clickreviews/tests/test_cr_systemd.py 2015-04-30 20:32:50.000000000 +0000 +++ click-reviewers-tools-0.28/clickreviews/tests/test_cr_systemd.py 2015-05-27 22:14:36.000000000 +0000 @@ -26,6 +26,19 @@ cr_tests.mock_patch() super() + def _create_ports(self, hook=False): + port = "port" + negotiable = "negotiable" + if hook: # handle weird formatting in .snappy-systemd + port = "Port" + negotiable = "Negotiable" + ports = {'internal': {'int1': {port: '8081/tcp', negotiable: True}}, + 'external': {'ext1': {port: '80/tcp', negotiable: False}, + 'ext2': {port: '88/udp'} + } + } + return ports + def _set_service(self, entries, name=None): d = dict() if name is None: @@ -121,7 +134,7 @@ c = ClickReviewSystemd(self.test_name) c.check_optional() r = c.click_report - expected_counts = {'info': 3, 'warn': 0, 'error': 0} + expected_counts = {'info': 5, 'warn': 0, 'error': 0} self.check_results(r, expected_counts) def test_check_optional_stop_empty(self): @@ -166,7 +179,7 @@ c = ClickReviewSystemd(self.test_name) c.check_optional() r = c.click_report - expected_counts = {'info': 3, 'warn': 0, 'error': 0} + expected_counts = {'info': 5, 'warn': 0, 'error': 0} self.check_results(r, expected_counts) def test_check_optional_stop_without_start(self): @@ -177,7 +190,7 @@ c = ClickReviewSystemd(self.test_name) c.check_optional() r = c.click_report - expected_counts = {'info': 3, 'warn': 0, 'error': 0} + expected_counts = {'info': 5, 'warn': 0, 'error': 0} self.check_results(r, expected_counts) def test_check_optional_stop_without_start2(self): @@ -191,7 +204,7 @@ c = ClickReviewSystemd(self.test_name) c.check_optional() r = c.click_report - expected_counts = {'info': 3, 'warn': 0, 'error': 0} + expected_counts = {'info': 5, 'warn': 0, 'error': 0} self.check_results(r, expected_counts) def test_check_unknown(self): @@ -826,3 +839,316 @@ r = c.click_report expected_counts = {'info': None, 'warn': 0, 'error': 1} self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_pkgname(self): + '''Test check_snappy_service_bus_name() - pkgname''' + name = self.test_name.split('_')[0] + self.set_test_pkg_yaml("name", name) + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", name)]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_appname(self): + '''Test check_snappy_service_bus_name() - appname''' + name = self.test_name.split('_')[0] + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "%s.%s" % (name, "test-app"))]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_pkgname_vendor(self): + '''Test check_snappy_service_bus_name() - pkgname with vendor''' + name = "foo" + self.set_test_pkg_yaml("name", name) + self.set_test_pkg_yaml("vendor", "f ") + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "com.isp.%s" % name)]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.is_snap = True + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_appname_vendor(self): + '''Test check_snappy_service_bus_name() - appname with vendor''' + name = "foo" + self.set_test_pkg_yaml("name", name) + self.set_test_pkg_yaml("vendor", "f ") + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "com.isp.%s.%s" % (name, "test-app"))]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.is_snap = True + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': 3, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_pkgname_bad(self): + '''Test check_snappy_service_bus_name() - bad pkgname''' + name = self.test_name.split('_')[0] + self.set_test_pkg_yaml("name", name) + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", name + "-bad")]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_appname_bad(self): + '''Test check_snappy_service_bus_name() - bad appname''' + name = self.test_name.split('_')[0] + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "%s.%s-bad" % (name, "test-app"))]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_pkgname_vendor_bad(self): + '''Test check_snappy_service_bus_name() - bad pkgname with vendor''' + name = "foo" + self.set_test_pkg_yaml("name", name) + self.set_test_pkg_yaml("vendor", "f ") + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "com.isp.%s-bad" % name)]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.is_snap = True + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_appname_vendor_bad(self): + '''Test check_snappy_service_bus_name() - bad appname with vendor''' + name = "foo" + self.set_test_pkg_yaml("name", name) + self.set_test_pkg_yaml("vendor", "f ") + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "com.isp.%s.%s-bad" % (name, + "test-app"))]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.is_snap = True + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_empty(self): + '''Test check_snappy_service_bus_name() - bad (empty)''' + name = self.test_name.split('_')[0] + self.set_test_pkg_yaml("name", name) + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "")]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_bus_name_bad_regex(self): + '''Test check_snappy_service_bus_name() - bad (regex)''' + name = self.test_name.split('_')[0] + self.set_test_pkg_yaml("name", name) + self._set_service([("start", "bin/test-app"), + ("description", "something"), + ("bus-name", "name$")]) + c = ClickReviewSystemd(self.test_name) + c.systemd_files['test-app'] = "meta/test-app.snappy-systemd" + c.check_snappy_service_bus_name() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 2} + self.check_results(r, expected_counts) + + def test_check_service_ports(self): + '''Test check_service_ports()''' + ports = self._create_ports(hook=True) + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_service_ports() + r = c.click_report + expected_counts = {'info': 8, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports(self): + '''Test check_snappy_service_ports()''' + ports = self._create_ports() + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': 8, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_internal(self): + '''Test check_snappy_service_ports() - internal''' + ports = self._create_ports() + del ports['internal'] + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': 6, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_external(self): + '''Test check_snappy_service_ports() - external''' + ports = self._create_ports() + del ports['external'] + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': 4, 'warn': 0, 'error': 0} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_empty(self): + '''Test check_snappy_service_ports() - empty''' + ports = self._create_ports() + del ports['internal'] + del ports['external'] + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_bad_key(self): + '''Test check_snappy_service_ports() - bad key''' + ports = self._create_ports() + ports['xternal'] = ports['external'] + del ports['external'] + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_missing_internal(self): + '''Test check_snappy_service_ports() - missing internal''' + ports = self._create_ports() + del ports['internal']['int1'] + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_missing_external(self): + '''Test check_snappy_service_ports() - missing external''' + ports = self._create_ports() + del ports['external']['ext1'] + del ports['external']['ext2'] + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_missing_external_subkey(self): + '''Test check_snappy_service_ports() - missing external subkey''' + ports = self._create_ports() + del ports['external']['ext2']['port'] + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_invalid_internal_subkey(self): + '''Test check_snappy_service_ports() - invalid internal subkey''' + ports = self._create_ports() + ports['internal']['int1']['prt'] = ports['internal']['int1']['port'] + del ports['internal']['int1']['port'] + del ports['internal']['int1']['negotiable'] + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_invalid_internal_port(self): + '''Test check_snappy_service_ports() - invalid internal port''' + ports = self._create_ports() + ports['internal']['int1']['port'] = "bad/8080" + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_invalid_internal_low_port(self): + '''Test check_snappy_service_ports() - invalid internal low port''' + ports = self._create_ports() + ports['internal']['int1']['port'] = "0/tcp" + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_invalid_internal_high_port(self): + '''Test check_snappy_service_ports() - invalid internal high port''' + ports = self._create_ports() + ports['internal']['int1']['port'] = "65536/tcp" + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) + + def test_check_snappy_service_ports_invalid_internal_negotiable(self): + '''Test check_snappy_service_ports() - invalid internal negotiable''' + ports = self._create_ports() + ports['internal']['int1']['negotiable'] = -99999999 + + self.set_test_systemd(self.default_appname, "ports", ports) + c = ClickReviewSystemd(self.test_name) + c.check_snappy_service_ports() + r = c.click_report + expected_counts = {'info': None, 'warn': 0, 'error': 1} + self.check_results(r, expected_counts) diff -Nru click-reviewers-tools-0.27/debian/changelog click-reviewers-tools-0.28/debian/changelog --- click-reviewers-tools-0.27/debian/changelog 2015-06-10 22:00:15.000000000 +0000 +++ click-reviewers-tools-0.28/debian/changelog 2015-06-10 21:07:42.000000000 +0000 @@ -1,3 +1,30 @@ +click-reviewers-tools (0.28) wily; urgency=medium + + [ Jamie Strandboge ] + * Makefile: perform run-pyflakes in check target + * cr_systemd.py: add bus-name checks and update testsuite + * add security yaml checks + * cr_lint.py: don't allow same key in 'binaries' and 'services' + * cr_lint.py: implement hashes.yaml checks + * update README + * cr_desktop.py: add check to help transition away from obsoleted + ubuntu-html5-app-launcher + * cr_common.py: remove snappy 'integration' checks + * cr_systemd.py: implement ports checks + * cr_systemd.py, cr_bin_path.py: error out if services or binaries is empty, + repectively + * cr_lint.py: update pkgname checks for snaps-- shouldn't have '.' in the + name + * cr_lint.py: add snappy-config checks + * cr_lint.py: maintainer isn't needed in compat click manifest for snaps + that don't specify vendor + * debian/control: Depends on binutils (for 'ar') + + [ Marcus Tomlinson ] + * cr_scope.py: add "keywords" to the list of optional scope .ini keys + + -- Jamie Strandboge Wed, 10 Jun 2015 16:07:33 -0500 + click-reviewers-tools (0.27) wily; urgency=medium * cr_security.py: add ubuntu-sdk-15.04 framework and policy version diff -Nru click-reviewers-tools-0.27/debian/control click-reviewers-tools-0.28/debian/control --- click-reviewers-tools-0.27/debian/control 2015-06-10 22:00:15.000000000 +0000 +++ click-reviewers-tools-0.28/debian/control 2015-06-09 12:59:02.000000000 +0000 @@ -22,7 +22,8 @@ Package: click-reviewers-tools Architecture: all -Depends: python3-apt, +Depends: binutils, + python3-apt, python3-debian, python3-lxml, python3-magic, diff -Nru click-reviewers-tools-0.27/Makefile click-reviewers-tools-0.28/Makefile --- click-reviewers-tools-0.27/Makefile 2015-04-30 20:32:49.000000000 +0000 +++ click-reviewers-tools-0.28/Makefile 2015-05-11 15:15:49.000000000 +0000 @@ -8,6 +8,7 @@ ./run-tests syntax-check: clean + ./run-pyflakes ./run-pep8 check: test syntax-check diff -Nru click-reviewers-tools-0.27/README click-reviewers-tools-0.28/README --- click-reviewers-tools-0.27/README 2015-04-30 20:32:50.000000000 +0000 +++ click-reviewers-tools-0.28/README 2015-05-20 14:06:42.000000000 +0000 @@ -1,13 +1,29 @@ Runnable tests: +- bin/click-check-bin-path: snappy bin-path tests +- bin/click-check-content-hub: content-hub hook tests +- bin/click-check-desktop: desktop hook tests +- bin/click-check-framework: click framework tests +- bin/click-check-functional: a few functional tests - bin/click-check-lint: lint tests +- bin/click-check-online-accounts: online accounts tests +- bin/click-check-push-helper: push-helper tests +- bin/click-check-scope: scope tests - bin/click-check-security: security hook tests -- bin/click-check-desktop: desktop hook tests +- bin/click-check-systemd: snappy systemd tests +- bin/click-check-url-dispatcher: url-dispatcher hook tests - bin/click-run-checks: all tests +This gives an alternate view on bin/click-run-checks: +- bin/click-review + +Running tests locally: +$ PYTHONPATH=$PWD ./bin/click-review /path/to/click + Importable tests: - clickreviews/cr_lint.py: lint tests - clickreviews/cr_security.py: security hook tests - clickreviews/cr_desktop.py: desktop hook tests +- ... In general, add or modify tests and report by using: self._add_result(, , ) @@ -27,7 +43,10 @@ * add tests to cr_.py. If you name the tests 'check_' ClickReview.do_checks() will enumerate and run them automatically -To run tests, just execute: run-tests +To run tests, just execute: +$ ./run-tests # all tests +$ ./run-tests test_cr_security.py # only security tests + If you are going to develop the tools regularly, you might want to add a bzr hook to run the testsuite before committing. Eg, add something like this to diff -Nru click-reviewers-tools-0.27/run-pep8 click-reviewers-tools-0.28/run-pep8 --- click-reviewers-tools-0.27/run-pep8 2015-04-30 20:32:49.000000000 +0000 +++ click-reviewers-tools-0.28/run-pep8 2015-05-18 21:08:42.000000000 +0000 @@ -2,8 +2,12 @@ set -e echo "= pep8 =" -for i in ./bin/update-* ./bin/click-check-* ./bin/click-show-files ./bin/click-review \ - ./clickreviews/*py ./clickreviews/tests/*py ; do +for i in ./clickreviews/*py \ + ./clickreviews/tests/*py \ + ./bin/update-* \ + ./bin/click-check-* \ + ./bin/click-show-files \ + ./bin/click-review ; do echo "Checking $i" pep8 $i done