diff -Nru charliecloud-0.9.6/bin/charliecloud.c charliecloud-0.9.10/bin/charliecloud.c --- charliecloud-0.9.6/bin/charliecloud.c 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/bin/charliecloud.c 2019-05-14 20:34:32.000000000 +0000 @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -98,6 +99,7 @@ void join_end(); void log_ids(const char *func, int line); bool path_exists(char *path); +unsigned long path_mount_flags(char *path); void path_split(char *path, char **dir, char **base); void sem_timedwait_relative(sem_t *sem, int timeout); void setup_namespaces(struct container *c); @@ -193,23 +195,24 @@ } // Container /home. if (!c->private_home) { - char *oldhome, *newhome; + char *newhome; // Mount tmpfs on guest /home because guest root is read-only tmpfs_mount("/home", c->newroot, "size=4m"); // Bind-mount user's home directory at /home/$USER. The main use case is // dotfiles. - oldhome = getenv("HOME"); - Tf (oldhome != NULL, "cannot find home directory: $HOME not set"); + Tf (c->old_home != NULL, "cannot find home directory: is $HOME set?"); newhome = cat("/home/", getenv("USER")); Z_ (mkdir(cat(c->newroot, newhome), 0755)); - bind_mount(oldhome, newhome, c->newroot, BD_REQUIRED, 0); + bind_mount(c->old_home, newhome, c->newroot, BD_REQUIRED, 0); } - // Bind-mount /usr/bin/ch-ssh if it exists. - if (path_exists(cat(c->newroot, "/usr/bin/ch-ssh"))) { + // Container /usr/bin/ch-ssh. + if (c->ch_ssh) { char chrun_file[PATH_CHARS]; int len = readlink("/proc/self/exe", chrun_file, PATH_CHARS); T_ (len >= 0); - chrun_file[ lennewroot, "/usr/bin/ch-ssh")), + "--ch-ssh: /usr/bin/ch-ssh not in image"); + chrun_file[ lennewroot, BD_REQUIRED, 0); } @@ -225,7 +228,11 @@ // Re-mount new root read-only unless --write or already read-only. if (!c->writable && !(access(c->newroot, W_OK) == -1 && errno == EROFS)) { - Zf (mount(NULL, c->newroot, NULL, MS_REMOUNT|MS_BIND|MS_RDONLY, NULL), + unsigned long flags = path_mount_flags(c->newroot) + | MS_REMOUNT // Re-mount ... + | MS_BIND // only this mount point ... + | MS_RDONLY; // read-only. + Zf (mount(NULL, c->newroot, NULL, flags, NULL), "can't re-mount image read-only (is it on NFS?)"); } // Pivot into the new root. Use /dev because it's available even in @@ -405,6 +412,38 @@ return false; } +/* Return the mount flags of the file system containing path, suitable for + passing to mount(2). + + This is messy because, the flags we get from statvfs(3) are ST_* while the + flags needed by mount(2) are MS_*. My glibc has a comment in bits/statvfs.h + that the ST_* "should be kept in sync with" the MS_* flags, and the values + do seem to match, but there are additional undocumented flags in there. + Also, the kernel contains a test "unprivileged-remount-test.c" that + manually translates the flags. Thus, I wasn't comfortable simply passing + the output of statvfs(3) to mount(2). */ +unsigned long path_mount_flags(char *path) +{ + struct statvfs sv; + unsigned long known_flags = ST_MANDLOCK | ST_NOATIME | ST_NODEV + | ST_NODIRATIME | ST_NOEXEC | ST_NOSUID + | ST_RDONLY | ST_RELATIME | ST_SYNCHRONOUS; + + Z_ (statvfs(path, &sv)); + Ze (sv.f_flag & ~known_flags, "unknown mount flags: 0x%lx %s", + sv.f_flag & ~known_flags, path); + + return (sv.f_flag & ST_MANDLOCK ? MS_MANDLOCK : 0) + | (sv.f_flag & ST_NOATIME ? MS_NOATIME : 0) + | (sv.f_flag & ST_NODEV ? MS_NODEV : 0) + | (sv.f_flag & ST_NODIRATIME ? MS_NODIRATIME : 0) + | (sv.f_flag & ST_NOEXEC ? MS_NOEXEC : 0) + | (sv.f_flag & ST_NOSUID ? MS_NOSUID : 0) + | (sv.f_flag & ST_RDONLY ? MS_RDONLY : 0) + | (sv.f_flag & ST_RELATIME ? MS_RELATIME : 0) + | (sv.f_flag & ST_SYNCHRONOUS ? MS_SYNCHRONOUS : 0); +} + /* Split path into dirname and basename. */ void path_split(char *path, char **dir, char **base) { @@ -533,6 +572,24 @@ Z_ (unlink(path)); } +/* Split string str at first instance of delimiter del. Set *a to the part + before del, and *b to the part after. Both can be empty; if no token is + present, set both to NULL. Unlike strsep(3), str is unchanged; *a and *b + point into a new buffer allocated with malloc(3). This has two + implications: (1) the caller must free(3) *a but not *b, and (2) the parts + can be rejoined by setting *(*b-1) to del. The point here is to provide an + easier wrapper for strsep(3). */ +void split(char **a, char **b, char *str, char del) +{ + char delstr[2] = { del, 0 }; + T_ (str != NULL); + str = strdup(str); + *b = str; + *a = strsep(b, delstr); + if (*b == NULL) + *a = NULL; +} + /* Mount a tmpfs at the given path. */ void tmpfs_mount(char *dst, char *newroot, char *data) { diff -Nru charliecloud-0.9.6/bin/charliecloud.h charliecloud-0.9.10/bin/charliecloud.h --- charliecloud-0.9.6/bin/charliecloud.h 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/bin/charliecloud.h 2019-05-14 20:34:32.000000000 +0000 @@ -65,6 +65,7 @@ struct container { struct bind *binds; + bool ch_ssh; // bind /usr/bin/ch-ssh? gid_t container_gid; uid_t container_uid; char *newroot; @@ -74,6 +75,7 @@ char *join_tag; // identifier for synchronized join bool private_home; bool private_tmp; + char *old_home; // host path to user's home directory (i.e. $HOME) bool writable; }; @@ -88,4 +90,5 @@ void containerize(struct container *c); void msg(int level, char *file, int line, int errno_, char *fmt, ...); void run_user_command(char *argv[], char *initial_dir); +void split(char **a, char **b, char *str, char del); void version(void); diff -Nru charliecloud-0.9.6/bin/ch-build2dir charliecloud-0.9.10/bin/ch-build2dir --- charliecloud-0.9.6/bin/ch-build2dir 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/bin/ch-build2dir 2019-05-14 20:34:32.000000000 +0000 @@ -1,41 +1,41 @@ -#!/bin/sh +#!/bin/bash libexec="$(cd "$(dirname "$0")" && pwd)" . "${libexec}/base.sh" # shellcheck disable=SC2034 usage=$(cat < "$tar" +docker_ export "$cid" | pv_ -s "$size" > "$tar" docker_ rm "$cid" > /dev/null -ls -lh "$tar" + +# Add the Docker environment variables in ./environment for later consumption +# by "ch-run --set-env". +# +# 1. mktemp(1) isn't POSIX, but it seemed very likely to be installed if +# Docker is, and I couldn't find a more portable way of securely creating +# temporary files. (In particular, I would have preferred to pipe in the +# data rather than creating and deleting a temporary file.) +# +# 2. Blocking factor 1 (-b1) for tar is a bug workaround. Without this switch, +# tar 1.26, which is in RHEL, corrupts the tarball instead of appending to +# it. This doesn't happen in 1.29 in Debian Stretch, and building GNU tar +# from Git source was too hard, so I couldn't bisect a specific commit that +# fixed the bug to learn what exactly was going on. (See PR #371.) +# +# 3. This assumes that the tarball from Docker does not have a single +# top-level directory (i.e., is a tarbomb). +# +echo "adding environment" +temp=$(mktemp --tmpdir ch-docker2tar.XXXXXX) +docker_ inspect "$image" --format='{{range .Config.Env}}{{println .}}{{end}}' \ + > "$temp" +tar rf "$tar" -b1 -P --xform="s|${temp}|environment|" "$temp" +rm "$temp" + +# Finish up. +echo "compressing" +cat "$tar" | pv_ -s "$size" | gzip_ -6 > "${tar}.gz" +rm "$tar" +ls -lh "${tar}.gz" diff -Nru charliecloud-0.9.6/bin/ch-fromhost charliecloud-0.9.10/bin/ch-fromhost --- charliecloud-0.9.6/bin/ch-fromhost 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/bin/ch-fromhost 2019-05-14 20:34:32.000000000 +0000 @@ -73,7 +73,7 @@ ) cray_mpi= # Cray fixups requested -#cray_openmpi= # ... for OpenMPI +cray_openmpi= # ... for OpenMPI cray_mpich= # ... for MPICH dest= image= @@ -228,17 +228,6 @@ [ "$image" ] || fatal "no image specified" -if [ $lib_found ]; then - # We want to put the libraries in the first directory that ldconfig - # searches, so that we can override (or overwrite) any of the same library - # that may already be in the image. - debug "asking ldconfig for shared library destination" - lib_dest=$( "${ch_bin}/ch-run" "$image" -- /sbin/ldconfig -Nv 2> /dev/null \ - | grep -E '^/' | cut -d: -f1 | head -1) - [ -z "${lib_dest%%/*}" ] || fatal "bad path from ldconfig: ${lib_dest}" - debug "shared library destination: ${lib_dest}" -fi - if [ $cray_mpi ]; then sentinel=/etc/opt/cray/release/cle-release [ -f $sentinel ] || fatal "not found: ${sentinel}: are you on a Cray?" @@ -249,8 +238,6 @@ cray_mpich=yes ;; *'Open MPI'*) - # FIXME: remove when implemented - # shellcheck disable=SC2034 cray_openmpi=yes ;; *) @@ -259,6 +246,24 @@ esac fi +if [ $lib_found ]; then + # We want to put the libraries in the first directory that ldconfig + # searches, so that we can override (or overwrite) any of the same library + # that may already be in the image. + debug "asking ldconfig for shared library destination" + # "ldconfig -Nv" gives some pointless warnings on stderr even if + # successful; we don't want to show those to users. However, we don't want + # to simply pipe stderr to /dev/null because this hides real errors. Thus, + # use the following abomination to pipe stdout and stderr to *separate + # grep commands*. See: https://stackoverflow.com/a/31151808 + lib_dest=$( { "${ch_bin}/ch-run" "$image" -- /sbin/ldconfig -Nv \ + 2>&1 1>&3 3>&- | grep -Ev '(^|dynamic linker, ignoring|given more than once)$' ; } \ + 3>&1 1>&2 | grep -E '^/' | cut -d: -f1 | head -1 ) + [ -n "$lib_dest" ] || fatal 'empty path from ldconfig' + [ -z "${lib_dest%%/*}" ] || fatal "bad path from ldconfig: ${lib_dest}" + debug "shared library destination: ${lib_dest}" +fi + if [ $lib_dest_print ]; then echo "$lib_dest" exit 0 @@ -299,21 +304,55 @@ queue_files /opt/cray/alps/default/lib64/libalpslli.so.0.0.0 queue_files /opt/cray/wlm_detect/default/lib64/libwlm_detect.so.0.0.0 #queue_files /opt/cray/alps/default/lib64/libalps.so.0.0.0 +fi - # libwlm_detect.so requires this file to be present. - queue_mkdir /etc/opt/cray/wlm_detect - queue_files /etc/opt/cray/wlm_detect/active_wlm /etc/opt/cray/wlm_detect +if [ $cray_openmpi ]; then + # Both MPI_ROOT and MPIHOME are the base of the OpenMPI install tree. + # We use MPIHOME because some users unset MPI_ROOT. + # + # Note also that the OpenMPI module name is not standardized. + [ "$MPIHOME" ] \ + || fatal "MPIHOME not set; is OpenMPI module loaded?" + + # Inject libmpi from the host + host_libmpi=${MPIHOME}/lib/libmpi.so + [ -f "$host_libmpi" ] \ + || fatal "not found: ${host_libmpi}; is OpenMPI module loaded?" + queue_files "$host_libmpi" + queue_files "$( ldd "$host_libmpi" \ + | grep -E "/usr|/opt" \ + | sed -E 's/^.+ => (.+) \(0x.+\)$/\1/')" + # This appears to be the only dependency in /lib64 that we can't get from + # glibc in the image. + queue_files /lib64/liblustreapi.so + + # Remove libmpi.so* from image. This works around ParaView + # dlopen(3)-related errors that we don't understand. + image_libmpis=$( "${ch_bin}/ch-run" "$image" -- /sbin/ldconfig -p \ + | sed -nr 's|^.* => (/.*/libmpi\.so([0-9.]+)?)$|\1|p') + [ "$image_libmpis" ] || fatal "can't find libmpi.so* in image" + for f in $image_libmpis; do + queue_unlink "$f" + queue_unlink "$("${ch_bin}/ch-run" "$image" -- readlink -f "$f")" + done +fi +if [ $cray_mpi ]; then # ALPS libraries require the contents of this directory to be present at # the same path as the host. Create the mount point here, then ch-run # bind-mounts it later. queue_mkdir /var/opt/cray/alps/spool - # Cray MPICH needs a pile of hugetlbfs filesystems at an arbitrary path - # (it searched /proc/mounts). ch-run bind-mounts to here later. + # libwlm_detect.so requires this file to be present. + queue_mkdir /etc/opt/cray/wlm_detect + queue_files /etc/opt/cray/wlm_detect/active_wlm /etc/opt/cray/wlm_detect + + # uGNI needs a pile of hugetlbfs filesystems at paths that are arbitrary + # but in a specific order in /proc/mounts. ch-run bind-mounts here later. queue_mkdir /var/opt/cray/hugetlbfs fi + [ "$inject_files" ] || fatal "empty file list" debug "injecting into image: ${image}" @@ -343,6 +382,13 @@ [ "$d" ] || fatal "no destination for: ${f}" [ -z "${d%%/*}" ] || fatal "not an absolute path: ${d}" [ -d "${image}${d}" ] || fatal "not a directory: ${image}${d}" + if [ ! -w "${image}${d}" ]; then + # Some images unpack with unwriteable directories; fix. This seems + # like a bit of a kludge to me, so I'd like to remove this special + # case in the future if possible. (#323) + info "${image}${d} not writeable; fixing" + chmod u+w "${image}${d}" || fatal "can't chmod u+w: ${image}${d}" + fi cp --dereference --preserve=all "$f" "${image}${d}" \ || fatal "cannot inject: ${f}" done diff -Nru charliecloud-0.9.6/bin/ch-run.c charliecloud-0.9.10/bin/ch-run.c --- charliecloud-0.9.6/bin/ch-run.c 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/bin/ch-run.c 2019-05-14 20:34:32.000000000 +0000 @@ -6,6 +6,7 @@ #define _GNU_SOURCE #include #include +#include #include #include #include @@ -13,7 +14,6 @@ #include "charliecloud.h" - /** Constants and macros **/ /* Environment variables used by --join parameters. */ @@ -43,28 +43,42 @@ const struct argp_option options[] = { { "bind", 'b', "SRC[:DST]", 0, "mount SRC at guest DST (default /mnt/0, /mnt/1, etc.)"}, - { "cd", 'c', "DIR", 0, "initial working directory in container"}, - { "gid", 'g', "GID", 0, "run as GID within container" }, - { "join", 'j', 0, 0, "use same container as peer ch-run" }, - { "join-pid", -5, "PID", 0, "join a namespace using a PID" }, - { "join-ct", -3, "N", 0, "number of ch-run peers (implies --join)" }, - { "join-tag", -4, "TAG", 0, "label for peer group (implies --join)" }, - { "no-home", -2, 0, 0, "do not bind-mount your home directory"}, - { "private-tmp", 't', 0, 0, "use container-private /tmp" }, - { "uid", 'u', "UID", 0, "run as UID within container" }, - { "verbose", 'v', 0, 0, "be more verbose (debug if repeated)" }, - { "version", 'V', 0, 0, "print version and exit" }, - { "write", 'w', 0, 0, "mount image read-write"}, + { "cd", 'c', "DIR", 0, "initial working directory in container"}, + { "ch-ssh", -8, 0, 0, "bind ch-ssh into image"}, + { "gid", 'g', "GID", 0, "run as GID within container" }, + { "join", 'j', 0, 0, "use same container as peer ch-run" }, + { "join-pid", -5, "PID", 0, "join a namespace using a PID" }, + { "join-ct", -3, "N", 0, "number of ch-run peers (implies --join)" }, + { "join-tag", -4, "TAG", 0, "label for peer group (implies --join)" }, + { "no-home", -2, 0, 0, "do not bind-mount your home directory"}, + { "private-tmp", 't', 0, 0, "use container-private /tmp" }, + { "set-env", -6, "FILE", 0, "set environment variables in FILE"}, + { "uid", 'u', "UID", 0, "run as UID within container" }, + { "unset-env", -7, "GLOB", 0, "unset environment variable(s)" }, + { "verbose", 'v', 0, 0, "be more verbose (debug if repeated)" }, + { "version", 'V', 0, 0, "print version and exit" }, + { "write", 'w', 0, 0, "mount image read-write"}, { 0 } }; +/* One possible future here is that fix_environment() ends up in charliecloud.c + and we add other actions such as SET, APPEND_PATH, etc. */ +enum env_action { END, SET_FILE, UNSET_GLOB }; // END must be zero + +struct env_delta { + enum env_action action; + char *arg; +}; + struct args { struct container c; + struct env_delta *env_deltas; char *initial_dir; }; /** Function prototypes **/ +void env_delta_append(struct env_delta **ds, enum env_action act, char *arg); void fix_environment(struct args *args); bool get_first_env(char **array, char **name, char **value); int join_ct(int cli_ct); @@ -77,12 +91,14 @@ /** Global variables **/ const struct argp argp = { options, parse_opt, args_doc, usage }; +extern char **environ; // see environ(7) /** Main **/ int main(int argc, char *argv[]) { + bool argp_help_fmt_set; struct args args; int arg_next; int c_argc; @@ -91,19 +107,32 @@ privs_verify_invoking(); T_ (args.c.binds = calloc(1, sizeof(struct bind))); + args.c.ch_ssh = false; args.c.container_gid = getegid(); args.c.container_uid = geteuid(); - args.initial_dir = NULL; args.c.join = false; - args.c.join_pid = 0; args.c.join_ct = 0; + args.c.join_pid = 0; args.c.join_tag = NULL; args.c.private_home = false; args.c.private_tmp = false; + args.c.old_home = getenv("HOME"); + args.c.writable = false; + T_ (args.env_deltas = calloc(1, sizeof(struct env_delta))); + args.initial_dir = NULL; verbose = 1; // in charliecloud.h - Z_ (setenv("ARGP_HELP_FMT", "opt-doc-col=25,no-dup-args-note", 0)); + /* I couldn't find a way to set argp help defaults other than this + environment variable. Kludge sets/unsets only if not already set. */ + if (getenv("ARGP_HELP_FMT")) + argp_help_fmt_set = true; + else { + argp_help_fmt_set = false; + Z_ (setenv("ARGP_HELP_FMT", "opt-doc-col=25,no-dup-args-note", 0)); + } Z_ (argp_parse(&argp, argc, argv, 0, &(arg_next), &args)); + if (!argp_help_fmt_set) + Z_ (unsetenv("ARGP_HELP_FMT")); Te (arg_next < argc - 1, "NEWROOT and/or CMD not specified"); args.c.newroot = realpath(argv[arg_next], NULL); @@ -127,8 +156,8 @@ INFO("join: %d %d %s %d", args.c.join, args.c.join_ct, args.c.join_tag, args.c.join_pid); INFO("private /tmp: %d", args.c.private_tmp); - containerize(&args.c); fix_environment(&args); + containerize(&args.c); run_user_command(c_argv, args.initial_dir); // should never return exit(EXIT_FAILURE); } @@ -136,31 +165,112 @@ /** Supporting functions **/ +/* Append a new env_delta to an existing null-terminated list. */ +void env_delta_append(struct env_delta **ds, enum env_action act, char *arg) +{ + int i; + + for (i = 0; (*ds)[i].action != END; i++) // count existing + ; + T_ (*ds = realloc(*ds, (i+2) * sizeof(struct env_delta))); + (*ds)[i+1].action = END; + (*ds)[i].action = act; + (*ds)[i].arg = arg; +} + /* Adjust environment variables. */ void fix_environment(struct args *args) { - char *old, *new; + char *name, *old_value, *new_value; // $HOME: Set to /home/$USER unless --no-home specified. - old = getenv("HOME"); if (!args->c.private_home) { - old = getenv("USER"); - if (old == NULL) { + old_value = getenv("USER"); + if (old_value == NULL) { WARNING("$USER not set; cannot rewrite $HOME"); } else { - T_ (1 <= asprintf(&new, "/home/%s", old)); - Z_ (setenv("HOME", new, 1)); + T_ (1 <= asprintf(&new_value, "/home/%s", old_value)); + Z_ (setenv("HOME", new_value, 1)); } } // $PATH: Append /bin if not already present. - old = getenv("PATH"); - if (old == NULL) { + old_value = getenv("PATH"); + if (old_value == NULL) { WARNING("$PATH not set"); - } else if (strstr(old, "/bin") != old && !strstr(old, ":/bin")) { - T_ (1 <= asprintf(&new, "%s:/bin", old)); - Z_ (setenv("PATH", new, 1)); - INFO("new $PATH: %s", new); + } else if ( strstr(old_value, "/bin") != old_value + && !strstr(old_value, ":/bin")) { + T_ (1 <= asprintf(&new_value, "%s:/bin", old_value)); + Z_ (setenv("PATH", new_value, 1)); + INFO("new $PATH: %s", new_value); + } + + // --set-env and --unset-env. + for (int i = 0; args->env_deltas[i].action != END; i++) { + char *arg = args->env_deltas[i].arg; + if (args->env_deltas[i].action == SET_FILE) { + FILE *fp; + Tf (fp = fopen(arg, "r"), "--set-env: can't open: %s", arg); + for (int j = 1; true; j++) { + char *line = NULL; + size_t len = 0; + errno = 0; + if (-1 == getline(&line, &len, fp)) { + if (errno == 0) // EOF + break; + else // error + Tf (0, "--set-env: error reading: %s", arg); + } + if (strlen(line) == 0 || line[0] == '\n') + continue; // skip empty line + if (line[strlen(line) - 1] == '\n') + line[strlen(line) - 1] = 0; // remove newline + split(&name, &new_value, line, '='); + Te (name != NULL, "--set-env: no delimiter: %s:%d", arg, j); + Te (strlen(name) != 0, "--set-env: empty name: %s:%d", arg, j); + if ( strlen(new_value) >= 2 + && new_value[0] == '\'' + && new_value[strlen(new_value) - 1] == '\'') { + new_value[strlen(new_value) - 1] = 0; // strip trailing quote + new_value++; // strip leading + } + INFO("environment: %s=%s", name, new_value); + Z_ (setenv(name, new_value, 1)); + } + fclose(fp); + } else { + T_ (args->env_deltas[i].action == UNSET_GLOB); + /* Removing variables from the environment is tricky, because there + is no standard library function to iterate through the + environment, and the environ global array can be re-ordered after + unsetenv(3) [1]. Thus, the only safe way without additional + storage is an O(n^2) search until no matches remain. + + It is legal to assign to environ [2]. We build up a copy, omitting + variables that match the glob, which is O(n), and then do so. + + [1]: https://unix.stackexchange.com/a/302987 + [2]: http://man7.org/linux/man-pages/man3/exec.3p.html */ + char **new_environ; + int old_i, new_i; + for (old_i = 0; environ[old_i] != NULL; old_i++) + ; + T_ (new_environ = calloc(old_i + 1, sizeof(char *))); + for (old_i = 0, new_i = 0; environ[old_i] != NULL; old_i++) { + int matchp; + split(&name, &old_value, environ[old_i], '='); + T_ (name != NULL); // env lines should always have equals + matchp = fnmatch(arg, name, 0); + if (!matchp) { + INFO("environment: unset %s", name); + } else { + T_ (matchp == FNM_NOMATCH); + *(old_value - 1) = '='; // rejoin line + new_environ[new_i++] = name; + } + } + environ = new_environ; + } } } @@ -269,6 +379,16 @@ case -5: // --join-pid args->c.join_pid = parse_int(arg, false, "--join-pid"); break; + case -6: // --set-env + env_delta_append(&(args->env_deltas), SET_FILE, arg); + break; + case -7: // --unset-env + Te (strlen(arg) > 0, "--unset-env: GLOB must have non-zero length"); + env_delta_append(&(args->env_deltas), UNSET_GLOB, arg); + break;; + case -8: // --ch-ssh + args->c.ch_ssh = true; + break; case 'c': args->initial_dir = arg; break; diff -Nru charliecloud-0.9.6/bin/ch-tar2dir charliecloud-0.9.10/bin/ch-tar2dir --- charliecloud-0.9.6/bin/ch-tar2dir 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/bin/ch-tar2dir 2019-05-14 20:34:32.000000000 +0000 @@ -1,11 +1,13 @@ #!/bin/sh +set -e + libexec="$(cd "$(dirname "$0")" && pwd)" . "${libexec}/base.sh" # shellcheck disable=SC2034 usage=$(cat <&2 + exit 1 +fi +if [ ! -d "${2}" ]; then + echo "can't unpack: ${2} is not a directory" 1>&2 + exit 1 +fi -# Is the tarball a regular file (or symlink) and readable? -if [ ! -f "$tarball" ] || [ ! -r "$tarball" ]; then - echo "can't read ${tarball}" 1>&2 +# Figure out the real tarball name. If the provided $1 already has a tar +# extension, just test that name; if not, also append the plausible extensions +# and try those too. +for ext in '' .tar.gz .tar.xz .tgz; do + c=${1}${ext} + if [ ! -f "$c" ] || [ ! -r "$c" ]; then + echo "can't read: ${c}" 1>&2 + case $1 in + *.tar.*|*.tgz) + break + ;; + *) + continue + ;; + esac + fi + tarball=$c + if [ -n "$ext" ]; then + echo "found: ${tarball}" 1>&2 + fi + # Infer decompression argument because GNU tar is unable to do so if input + # is a pipe, and we want to keep PV. See: + # https://www.gnu.org/software/tar/manual/html_section/tar_68.html + case $tarball in + *.tar.gz) + newroot=${2}/$(basename "${tarball%.tar.gz}") + decompress=z + ;; + *.tar.xz) + newroot=${2}/$(basename "${tarball%.tar.xz}") + decompress=J + ;; + *.tgz) + newroot=${2}/$(basename "${tarball%.tgz}") + decompress=z + ;; + *) + echo "unknown extension: ${tarball}" 1>&2 + exit 1 + ;; + esac + break +done +if [ -z "$tarball" ]; then + echo "no input found" 1>&2 exit 1 fi @@ -49,18 +97,34 @@ fi mkdir "$newroot" -echo 'This directory is a Charliecloud container image.' > "${newroot}/${sentinel}" # Use a pipe because PV ignores arguments if it's cat rather than PV. +# +# See FAQ on /dev exclusion. --no-wildcards-match-slash is needed to prevent * +# matching multiple directories; the tar default differs from sh behavior. size=$(stat -c%s "$tarball") pv_ -s "$size" < "$tarball" \ -| gzip_ -dc \ -| tar x$verbose -C "$newroot" -f - \ - --anchored --exclude='dev/*' --exclude='./dev/*' +| tar x$decompress -C "$newroot" -f - \ + --anchored --no-wildcards-match-slash \ + --exclude='dev/*' --exclude='*/dev/*' # Make all directories writeable so we can delete image later (hello, Red Hat). find "$newroot" -type d -a ! -perm /200 -exec chmod u+w {} + +# If tarball had a single containing directory, move the contents up a level +# and remove the containing directory. It is non-trivial in POSIX sh to deal +# with hidden files; see https://unix.stackexchange.com/a/6397. +files=$(ls -Aq "$newroot") +if [ "$(echo "$files" | wc -l)" -eq 1 ]; then + ( cd "${newroot}/${files}" + for f in * .[!.]* ..?*; do + if [ -e "$f" ]; then mv -- "$f" ..; fi + done ) + rmdir "${newroot}/${files}" +fi + # Ensure directories that ch-run needs exist. -mkdir -p "${newroot}/dev" +echo 'This directory is a Charliecloud image.' > "${newroot}/${sentinel}" +mkdir -p "${newroot}/dev" "${newroot}/etc" "${newroot}/proc" "${newroot}/sys" +touch "${newroot}/etc/hosts" "${newroot}/etc/resolv.conf" for i in $(seq 0 9); do mkdir -p "${newroot}/mnt/${i}"; done echo "${newroot} unpacked ok" diff -Nru charliecloud-0.9.6/debian/changelog charliecloud-0.9.10/debian/changelog --- charliecloud-0.9.6/debian/changelog 2018-12-14 19:09:40.000000000 +0000 +++ charliecloud-0.9.10/debian/changelog 2019-09-08 18:23:14.000000000 +0000 @@ -1,3 +1,17 @@ +charliecloud (0.9.10-1) unstable; urgency=medium + + * New upstream version 0.9.10 + * Update copyright years + * Bump debhelper compat level to 12 and switch to debhelper-compat + * Adjust debhelper config files to upstream's new directory structure + * Switch to Standards-Version 4.4.0 (no changes required) + * Add d/salsa-ci.yml file + * Switch to python3 (closes: #936290) + - Add patch to switch to python3 + - d/control: Switch to python3 build dependencies + + -- Peter Wienemann Sun, 08 Sep 2019 20:23:14 +0200 + charliecloud (0.9.6-1) unstable; urgency=medium * New upstream version 0.9.6 diff -Nru charliecloud-0.9.6/debian/charliecloud-doc.examples charliecloud-0.9.10/debian/charliecloud-doc.examples --- charliecloud-0.9.6/debian/charliecloud-doc.examples 2018-12-14 19:09:40.000000000 +0000 +++ charliecloud-0.9.10/debian/charliecloud-doc.examples 2019-09-08 18:23:14.000000000 +0000 @@ -1 +1 @@ -examples/* +usr/lib/charliecloud/examples/* diff -Nru charliecloud-0.9.6/debian/charliecloud.install charliecloud-0.9.10/debian/charliecloud.install --- charliecloud-0.9.6/debian/charliecloud.install 2018-12-14 19:09:40.000000000 +0000 +++ charliecloud-0.9.10/debian/charliecloud.install 2019-09-08 18:23:14.000000000 +0000 @@ -1,2 +1,2 @@ debian/tmp/usr/bin -debian/tmp/usr/lib +debian/tmp/usr/lib/charliecloud/*.sh diff -Nru charliecloud-0.9.6/debian/compat charliecloud-0.9.10/debian/compat --- charliecloud-0.9.6/debian/compat 2018-12-14 19:09:40.000000000 +0000 +++ charliecloud-0.9.10/debian/compat 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -11 diff -Nru charliecloud-0.9.6/debian/control charliecloud-0.9.10/debian/control --- charliecloud-0.9.6/debian/control 2018-12-14 19:09:40.000000000 +0000 +++ charliecloud-0.9.10/debian/control 2019-09-08 18:23:14.000000000 +0000 @@ -3,13 +3,13 @@ Priority: optional Maintainer: Debian HPC Team Uploaders: Lucas Nussbaum , Peter Wienemann -Build-Depends: debhelper (>= 11~), +Build-Depends: debhelper-compat (= 12), po-debconf, - python, - python-sphinx, - python-sphinx-rtd-theme, + python3, + python3-sphinx, + python3-sphinx-rtd-theme, rsync -Standards-Version: 4.2.1 +Standards-Version: 4.4.0 Homepage: https://hpc.github.io/charliecloud/ Vcs-Git: https://salsa.debian.org/hpc-team/charliecloud.git Vcs-Browser: https://salsa.debian.org/hpc-team/charliecloud/tree/master diff -Nru charliecloud-0.9.6/debian/copyright charliecloud-0.9.10/debian/copyright --- charliecloud-0.9.6/debian/copyright 2018-12-14 19:09:40.000000000 +0000 +++ charliecloud-0.9.10/debian/copyright 2019-09-08 18:23:14.000000000 +0000 @@ -3,12 +3,12 @@ Source: https://github.com/hpc/charliecloud Files: * -Copyright: 2014-2018, Los Alamos National Security, LLC. +Copyright: 2014-2019, Los Alamos National Security, LLC. License: Apache-2.0-LANS Files: debian/* Copyright: 2017-2018 Lucas Nussbaum - 2018 Peter Wienemann + 2018-2019 Peter Wienemann License: Apache-2.0 License: Apache-2.0-LANS diff -Nru charliecloud-0.9.6/debian/patches/series charliecloud-0.9.10/debian/patches/series --- charliecloud-0.9.6/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/debian/patches/series 2019-09-08 18:23:14.000000000 +0000 @@ -0,0 +1 @@ +use-python3.patch diff -Nru charliecloud-0.9.6/debian/patches/use-python3.patch charliecloud-0.9.10/debian/patches/use-python3.patch --- charliecloud-0.9.6/debian/patches/use-python3.patch 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/debian/patches/use-python3.patch 2019-09-08 18:23:14.000000000 +0000 @@ -0,0 +1,30 @@ +From: Peter Wienemann +Date: Sun, 8 Sep 2019 19:33:21 +0200 +Subject: use python3 +Bug-Debian: https://bugs.debian.org/936290 + +--- + test/make-auto | 2 +- + test/make-perms-test | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/test/make-auto b/test/make-auto +index a6142ea..55d30b6 100755 +--- a/test/make-auto ++++ b/test/make-auto +@@ -1,4 +1,4 @@ +-#!/usr/bin/env python ++#!/usr/bin/env python3 + + from __future__ import print_function + +diff --git a/test/make-perms-test b/test/make-perms-test +index c4a4677..e893d7f 100755 +--- a/test/make-perms-test ++++ b/test/make-perms-test +@@ -1,4 +1,4 @@ +-#!/usr/bin/env python ++#!/usr/bin/env python3 + + # This script sets up a test directory for testing filesystem permissions + # enforcement in UDSS such as virtual machines and containers. It must be run diff -Nru charliecloud-0.9.6/debian/rules charliecloud-0.9.10/debian/rules --- charliecloud-0.9.6/debian/rules 2018-12-14 19:09:40.000000000 +0000 +++ charliecloud-0.9.10/debian/rules 2019-09-08 18:23:14.000000000 +0000 @@ -21,8 +21,10 @@ override_dh_auto_install: dh_auto_install dh_installdebconf - rm -rf debian/tmp/usr/share/doc/charliecloud/test - rm debian/tmp/usr/share/doc/charliecloud/LICENSE + # Do not put architecture dependent binaries in + # architecture independent charliecloud-doc package + rm debian/tmp/usr/lib/charliecloud/examples/syscalls/pivot_root + rm debian/tmp/usr/lib/charliecloud/examples/syscalls/userns # Patch out bundled jquery.js rm html/_static/jquery.js ln -s /usr/share/javascript/jquery/jquery.js html/_static/jquery.js diff -Nru charliecloud-0.9.6/debian/salsa-ci.yml charliecloud-0.9.10/debian/salsa-ci.yml --- charliecloud-0.9.6/debian/salsa-ci.yml 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/debian/salsa-ci.yml 2019-09-08 18:23:14.000000000 +0000 @@ -0,0 +1,7 @@ +--- +include: + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml + +variables: + RELEASE: 'unstable' diff -Nru charliecloud-0.9.6/doc-src/ch-build2dir_desc.rst charliecloud-0.9.10/doc-src/ch-build2dir_desc.rst --- charliecloud-0.9.6/doc-src/ch-build2dir_desc.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/ch-build2dir_desc.rst 2019-05-14 20:34:32.000000000 +0000 @@ -3,34 +3,45 @@ :: - $ ch-build2dir CONTEXT DEST [ARGS ...] + $ ch-build2dir -t TAG [ARGS ...] CONTEXT OUTDIR Description =========== -Build a Docker image as specified by the file :code:`Dockerfile` in the -current working directory and context directory :code:`CONTEXT`. Unpack it in -:code:`DEST`. - -Sudo privileges are required to run the :code:`docker` command. - -This runs the following command sequence: :code:`ch-build`, -:code:`ch-docker2tar`, and :code:`ch-tar2dir` but provides less flexibility -than the individual commands. +Build a Docker image named :code:`TAG` described by a Dockerfile (default +:code:`./Dockerfile`) and unpack it into :code:`OUTDIR/TAG`. This is a wrapper +for :code:`ch-build`, :code:`ch-docker2tar`, and :code:`ch-tar2dir`; see also +those man pages. Arguments: + :code:`ARGS` + additional arguments passed to :code:`ch-build` + :code:`CONTEXT` Docker context directory - :code:`DEST` - directory in which to place image tarball and directory + :code:`OUTDIR` + directory in which to place image directory (named :code:`TAG`) and + temporary tarball - :code:`ARGS` - additional arguments passed to :code:`ch-build` + :code:`-t TAG` + name (tag) of Docker image to build :code:`--help` print help and exit :code:`--version` print version and exit + +Examples +======== + +To build using :code:`./Dockerfile` and create image directory +:code:`/var/tmp/foo`:: + + $ ch-build2dir -t foo . /var/tmp + +Same as above, but build with a different Dockerfile:: + + $ ch-build2dir -t foo -f ./Dockerfile.foo . /var/tmp diff -Nru charliecloud-0.9.6/doc-src/ch-docker2tar_desc.rst charliecloud-0.9.10/doc-src/ch-docker2tar_desc.rst --- charliecloud-0.9.6/doc-src/ch-docker2tar_desc.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/ch-docker2tar_desc.rst 2019-05-14 20:34:32.000000000 +0000 @@ -11,6 +11,10 @@ Flattens the Docker image tagged :code:`IMAGE` into a Charliecloud tarball in directory :code:`OUTDIR`. +The Docker environment (e.g., :code:`ENV` statements) is placed in a file in +the tarball at :code:`./environment`, in a form suitable for :code:`ch-run +--set-env`. + Sudo privileges are required to run :code:`docker export`. Additional arguments: diff -Nru charliecloud-0.9.6/doc-src/ch-fromhost_desc.rst charliecloud-0.9.10/doc-src/ch-fromhost_desc.rst --- charliecloud-0.9.6/doc-src/ch-fromhost_desc.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/ch-fromhost_desc.rst 2019-05-14 20:34:32.000000000 +0000 @@ -92,8 +92,8 @@ Print version and exit. -:code:`--cray-mpi` prerequisites and quirks -=========================================== +:code:`--cray-mpi` dependencies and quirks +========================================== The implementation of :code:`--cray-mpi` for MPICH is messy, foul smelling, and brittle. It replaces or overrides the open source MPICH libraries diff -Nru charliecloud-0.9.6/doc-src/ch-run_desc.rst charliecloud-0.9.10/doc-src/ch-run_desc.rst --- charliecloud-0.9.6/doc-src/ch-run_desc.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/ch-run_desc.rst 2019-05-14 20:34:32.000000000 +0000 @@ -18,6 +18,9 @@ :code:`-c`, :code:`--cd=DIR` initial working directory in container + :code:`--ch-ssh` + bind :code:`ch-ssh(1)` into container at :code:`/usr/bin/ch-ssh` + :code:`-g`, :code:`--gid=GID` run as group :code:`GID` within container @@ -42,9 +45,15 @@ use container-private :code:`/tmp` (by default, :code:`/tmp` is shared with the host) + :code:`--set-env=FILE` + set environment variables as specified in host path :code:`FILE` + :code:`-u`, :code:`--uid=UID` run as user :code:`UID` within container + :code:`--unset-env=GLOB` + unset environment variables whose names match :code:`GLOB` + :code:`-v`, :code:`--verbose` be more verbose (debug if repeated) @@ -158,9 +167,24 @@ Environment variables ===================== -:code:`ch-run` generally tries to leave environment variables unchanged, but -in some cases, guests can be significantly broken unless environment variables -are tweaked. This section lists those changes. +:code:`ch-run` leaves environment variables unchanged, i.e. the host +environment is passed through unaltered, except: + +* limited tweaks to avoid significant guest breakage; +* user-set variables via :code:`--set-env`; and +* user-unset variables via :code:`--unset-env`. + +This section describes these features. + +The default tweaks happen first, and then :code:`--set-env` and +:code:`--unset-env` in the order specified on the command line. The latter two +can be repeated arbitrarily many times, e.g. to add/remove multiple variable +sets or add only some variables in a file. + +Default behavior +---------------- + +By default, :code:`ch-run` makes the following environment variable changes: * :code:`$HOME`: If the path to your home directory is not :code:`/home/$USER` on the host, then an inherited :code:`$HOME` will be incorrect inside the @@ -184,7 +208,158 @@ * `Fedora `_ * `Debian `_ +Setting variables with :code:`--set-env` +---------------------------------------- +The purpose of :code:`--set-env=FILE` is to set environment variables that +cannot be inherited from the host shell, e.g. Dockerfile :code:`ENV` +directives or other build-time configuration. :code:`FILE` is a host path to +provide the greatest flexibility; guest paths can be specified by prepending +the image path. + +:code:`ch-docker2tar(1)` lists variables specified at build time in +Dockerfiles in the image in file :code:`/environment`. To set these variables: +:code:`--set-env=$IMG/environment`. + +Variable values in :code:`FILE` replace any already set. If a variable is +repeated, the last value wins. + +The syntax of :code:`FILE` is key-value pairs separated by the first equals +character (:code:`=`, ASCII 61), one per line, with optional single straight +quotes (:code:`'`, ASCII 39) around the value. Empty lines are ignored. +Newlines (ASCII 10) are not permitted in either key or value. No variable +expansion, comments, etc. are provided. The value may be empty, but not the +key. (This syntax is designed to accept the output of :code:`printenv` and be +easily produced by other simple mechanisms.) Examples of valid lines: + +.. list-table:: + :header-rows: 1 + + * - Line + - Key + - Value + * - :code:`FOO=bar` + - :code:`FOO` + - :code:`bar` + * - :code:`FOO=bar=baz` + - :code:`FOO` + - :code:`bar=baz` + * - :code:`FLAGS=-march=foo -mtune=bar` + - :code:`FLAGS` + - :code:`-march=foo -mtune=bar` + * - :code:`FLAGS='-march=foo -mtune=bar'` + - :code:`FLAGS` + - :code:`-march=foo -mtune=bar` + * - :code:`FOO=` + - :code:`FOO` + - (empty string) + * - :code:`FOO=''` + - :code:`FOO` + - (empty string) + * - :code:`FOO=''''` + - :code:`FOO` + - :code:`''` (two single quotes) + +Example invalid lines: + +.. list-table:: + :header-rows: 1 + + * - Line + - Problem + * - :code:`FOO bar` + - no separator + * - :code:`=bar` + - key cannot be empty + +Example valid lines that are probably not what you want: + +.. Note: Plain leading space screws up ReST parser. We use ZERO WIDTH SPACE + U+200B, then plain space. This will copy and paste incorrectly, but that + seems unlikely. + +.. list-table:: + :header-rows: 1 + + * - Line + - Key + - Value + - Problem + * - :code:`FOO="bar"` + - :code:`FOO` + - :code:`"bar"` + - double quotes aren't stripped + * - :code:`FOO=bar # baz` + - :code:`FOO` + - :code:`bar # baz` + - comments not supported + * - :code:`PATH=$PATH:/opt/bin` + - :code:`PATH` + - :code:`$PATH:/opt/bin` + - variables not expanded + * - :code:`​ FOO=bar` + - :code:`​ FOO` + - :code:`bar` + - leading space in key + * - :code:`FOO= bar` + - :code:`FOO` + - :code:`​ bar` + - leading space in value + +Removing variables with :code:`--unset-env` +------------------------------------------- + +The purpose of :code:`--unset-env=GLOB` is to remove unwanted environment +variables. The argument :code:`GLOB` is a glob pattern (`dialect +`_ :code:`fnmatch(3)` +with no flags); all variables with matching names are removed from the +environment. + +.. warning:: + + Because the shell also interprets glob patterns, if any wildcard characters + are in :code:`GLOB`, it is important to put it in single quotes to avoid + surprises. + +:code:`GLOB` must be a non-empty string. + +Example 1: Remove the single environment variable :code:`FOO`:: + + $ export FOO=bar + $ env | fgrep FOO + FOO=bar + $ ch-run --unset-env=FOO $CH_TEST_IMGDIR/chtest -- env | fgrep FOO + $ + +Example 2: Hide from a container the fact that it's running in a Slurm +allocation, by removing all variables beginning with :code:`SLURM`. You might +want to do this to test an MPI program with one rank and no launcher:: + + $ salloc -N1 + $ env | egrep '^SLURM' | wc + 44 44 1092 + $ ch-run $CH_TEST_IMGDIR/mpihello-openmpi -- /hello/hello + [... long error message ...] + $ ch-run --unset-env='SLURM*' $CH_TEST_IMGDIR/mpihello-openmpi -- /hello/hello + 0: MPI version: + Open MPI v3.1.3, package: Open MPI root@c897a83f6f92 Distribution, ident: 3.1.3, repo rev: v3.1.3, Oct 29, 2018 + 0: init ok cn001.localdomain, 1 ranks, userns 4026532530 + 0: send/receive ok + 0: finalize ok + +Example 3: Clear the environment completely (remove all variables):: + + $ ch-run --unset-env='*' $CH_TEST_IMGDIR/chtest -- env + $ + +Note that some programs, such as shells, set some environment variables even +if started with no init files:: + + $ ch-run --unset-env='*' $CH_TEST_IMGDIR/debian9 -- bash --noprofile --norc -c env + SHLVL=1 + PWD=/ + _=/usr/bin/env + $ Examples ======== @@ -198,3 +373,5 @@ Run an MPI job that can use CMA to communicate:: $ srun ch-run --join /data/foo -- bar + +.. LocalWords: mtune diff -Nru charliecloud-0.9.6/doc-src/ch-tar2dir_desc.rst charliecloud-0.9.10/doc-src/ch-tar2dir_desc.rst --- charliecloud-0.9.6/doc-src/ch-tar2dir_desc.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/ch-tar2dir_desc.rst 2019-05-14 20:34:32.000000000 +0000 @@ -10,12 +10,14 @@ Extract the tarball :code:`TARBALL` into a subdirectory of :code:`DIR`. :code:`TARBALL` must contain a Linux filesystem image, e.g. as created by -:code:`ch-docker2tar`. +:code:`ch-docker2tar`, and be compressed with :code:`gzip` or :code:`xz`. If +:code:`TARBALL` has no extension, try appending :code:`.tar.gz` and +:code:`.tar.xz`. Inside :code:`DIR`, a subdirectory will be created whose name corresponds to -the name of the tarball with the :code:`.tar.gz` suffix removed. If such a -directory exists already and appears to be a Charliecloud container image, it -is removed and replaced. If the existing directory doesn't appear to be a +the name of the tarball with :code:`.tar.gz` or other suffix removed. If such +a directory exists already and appears to be a Charliecloud container image, +it is removed and replaced. If the existing directory doesn't appear to be a container image, the script aborts with an error. Additional arguments: @@ -23,9 +25,6 @@ :code:`--help` print help and exit - :code:`--verbose` - be more verbose - :code:`--version` print version and exit diff -Nru charliecloud-0.9.6/doc-src/command-usage.rst charliecloud-0.9.10/doc-src/command-usage.rst --- charliecloud-0.9.6/doc-src/command-usage.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/command-usage.rst 2019-05-14 20:34:32.000000000 +0000 @@ -4,6 +4,10 @@ This section is a comprehensive description of the usage and arguments of the Charliecloud commands. Its content is identical to the commands' man pages. +.. contents:: + :depth: 1 + :local: + .. WARNING: The one-line summaries below are duplicated in list man_pages in conf.py. Any updates need to be made there also. @@ -23,7 +27,7 @@ ch-build2dir ++++++++++++ -Build a Charliecloud image from Dockerfile and unpack it. +Build a Charliecloud image and unpack it into a directory in one command. .. include:: ./ch-build2dir_desc.rst diff -Nru charliecloud-0.9.6/doc-src/conf.py charliecloud-0.9.10/doc-src/conf.py --- charliecloud-0.9.6/doc-src/conf.py 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/conf.py 2019-05-14 20:34:32.000000000 +0000 @@ -21,7 +21,7 @@ # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +needs_sphinx = '1.4.9' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. diff -Nru charliecloud-0.9.6/doc-src/dev.rst charliecloud-0.9.10/doc-src/dev.rst --- charliecloud-0.9.6/doc-src/dev.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/dev.rst 2019-05-14 20:34:32.000000000 +0000 @@ -22,59 +22,12 @@ Workflow ======== -Branching model ---------------- - -* We try to keep the branching model simple. Right now, we're pretty similar - to Scott Chacon's “`GitHub Flow - `_”: Master is stable; - work on short-lived topic branches; use pull requests to ask for merging. - -* Tagged versions currently get more testing. We are working to improve - testing for normal commits on the master, but full parity is probably - unlikely. - -* Don't work directly on master. Even the project lead doesn't do this. While - it may appear that some trivial fixes are being committed to the master - directly, what's really happening is that these are prototyped on a branch - and then fast-forward merged after Travis passes. - -* Keep history tidy. While it's often difficult to avoid a branch history with - commits called "try 2" and "fix Travis", clean it up before submitting a PR. - Interactive rebase is your friend. - -* Feature branches should generally be rebased, rather than merged into, in - order to track master. PRs with conflicts will generally not be merged. - -* Feature branches are merged with either a merge commit or squash and rebase, - which squashes all the branch's commits into one and then rebases that - commit onto master's HEAD (called "squash and merge" by GitHub). This can be - done either on the command line or in the GitHub web interface. - - * Merge commit message example: - :code:`merge PR #268 from @j-ogas: remove ch-docker-run (closes #258)` - * Squash and rebase commit message example: - :code:`PR #270 from me: document locations of .bats files` - -* Feature branches in the main repo are deleted by the project lead after - merging. - -* Remove obsolete remote branches from your repo with :code:`git fetch --prune - --all`. - -Issues, pull requests, and milestones -------------------------------------- - -* We use milestones to organize what is planned for, and what actually - happened, in each version. - -* All but the most trivial changes should have an issue or a `pull request - `_ (PR). - The relevant commit message should close the issue (not PR) using the - `GitHub syntax - `_. +We try to keep procedures and the Git branching model simple. Right now, we're +pretty similar to Scott Chacon's “`GitHub Flow +`_”: Master is stable; +work on short-lived topic branches; use pull requests to ask for merging; keep issues organized with tags and milestones. -* The standard workflow is: +The standard workflow is: 1. Propose a change in an issue. @@ -83,71 +36,209 @@ 3. Get consensus on what to do and how to do it, with key information recorded in the issue. - 4. Assign the issue to a milestone. + 4. Submit a PR that refers to the issue. - 5. Submit a PR that refers to the issue. + 5. Assign the issue to a milestone. 6. Review/iterate. - 7. Project lead merges. No one other than the project lead should be - merging to or committing on master. - - Don't tag or milestone the PR in this case, so that the change is only - listed once in the various views. - -* Bare PRs with no corresponding issue are also considered but should have - reached consensus using other means, which should be stated in the PR. Tag - and milestone the PR. - -* We acknowledge submitted issues by tagging them. - -* Issues and PRs should address a single concern. If there are multiple - concerns, make separate issues and/or PRs. For example, PRs should not tidy - unrelated code. - -* Best practice for non-trivial changes is to draft documentation and/or - tests, get feedback on that, and then implement. - -* If you are assigned an issue, that means you are actively working on it or - will do so in the near future. "I'll get to this later" should not be - assigned to you. - -* PR review: - - * If you think you're done and it's ready to merge: Tag the PR :code:`ready - to merge`. Don't request review from the project lead. - - * If you think you're done and want review from someone other than the - project lead: Request review from that person using the GitHub web - interface. Once the PR passes review, go to the previous item. + 7. Project lead merges. - * If you're not done but want feedback: Request review from the person you - want to review, which can be the project lead. +Core team members may deliberate in public on GitHub or internally, whichever +they are comfortable with, making sure to follow LANL policy and taking into +account the probable desires of the recipient as well. + +Milestones +---------- + +We use milestones to organize what we plan to do next and what happened in a +given release. There are two groups of milestones: + +* :code:`next` contains the issues that we plan to complete soon but have not + yet landed on a specific release. Generally, we avoid putting PRs in here + because of their ticking clocks. + +* Each release has a milestone. These are dated with the target date for that + release. We put an issue in when it has actually landed in that release or + we are willing to delay that release until it does. We put a PR in when we + think it's reasonably likely to be merged for that release. + +If an issue is assigned to a person, that means they are actively leading the +work on it or will do so in the near future. Typically this happens when the +issue ends up in :code:`next`. Issues in a status of "I'll get to this later" +should not be assigned to a person. + +Peer review +----------- + +**Issues and pull requests.** The standard workflow is to introduce a +change in an issue, get consensus on what to do, and then create a `pull +request `_ +(PR) for the implementation. The issue, not the PR, should be tagged and +milestoned so a given change shows up only once in the various views. + +If consensus is obtained through other means (e.g., in-person discussion), +then open a PR directly. In this case, the PR should be tagged and milestoned, +since there is no issue. + +Trivial changes (e.g., fix Travis, fix a regression within a release, +code formatting) can be done without an issue or PR. + +**Address a single concern.** When possible, issues and PRs should address +completely one self-contained change. If there are multiple concerns, make +separate issues and/or PRs. For example, PRs should not tidy unrelated code, +and non-essential complications should be split into a follow-on issue. + +**Documentation and tests first.** The best practice for significant changes +is to draft documentation and/or tests first, get feedback on that, and then +implement the code. Reviews of the form "you need a completely different +approach" are no fun. + +**Tests must pass.** PRs will not be merged until they pass the tests. While +this most saliently includes Travis, the tests should also pass on your +development box as well as all relevant clusters (if appropriate for the +changes). + +**No close keywords in PRs.** While GitHub will interpret issue-closing +keywords (variations on `"closes", "fixes", and "resolves" +`_) in PR +descriptions, don't use this feature, because often the specific issues a PR +closes change over time, and we don't want to have to edit the description to +deal with that. We also want this information in only one place (the commit +log). Instead, use "addresses", and we'll edit the keywords into the commit +message(s) at merge time if needed. + +**PR review procedure.** When your PR is ready for review — which may or may +not be when you want it considered for merging — do one or both of: + +* Request review from the person(s) you want to look at it. If you think it + may be ready for merge, that should include the project lead. The purpose of + requsting review is so the person is notified you need their help. + +* If you think it may be ready to merge (even if you're not sure), then also + tag the PR :code:`ready to merge`. The purpose of this is so the project + lead can see which PRs are ready to consider for merging. If the project + lead decides it's ready, they will merge; otherwise, they'll untag. + +In both cases, the person from whom you requested review now owns the branch, +and you should stop work on it unless and until you get it back. + +Do not hesitate to pester your reviewer if you haven't heard back promptly. + +*Special case 1:* Often, the review consists of code changes, and the reviewer +will want you to assess those changes. GitHub doesn't let you request review +from the PR submitter, so this must be done with a comment, either online or +offline. + +*Special case 2:* GitHub will not let you request review from external people, +so this needs to be done with a comment too. Generally you should ask the +original bug reporter to review, to make sure it solves their problem. + +**Use multi-comment reviews.** Review comments should all be packaged up into +a single review; click *Start a review* rather than *Add single comment*. Then +the PR author gets only a single notification instead of one for every comment +you make, and it's clear when they branch is theirs again. + +Branching and merging +--------------------- + +**Don't commit directly to master.** Even the project lead doesn't do this. +While it may appear that some trivial fixes are being committed to the master +directly, what's really happening is that these are prototyped on a branch and +then fast-forward merged after the tests pass. + +**Merging to master.** Only the project lead should do this. + +**Branch merge procedure.** Generally, branches are merged in the GitHub web +interface with the *Squash and merge* button, which is :code:`git merge +--squash` under the hood. This squashes the branch into a single commit on +master. Commit message example:: + + PR #268 from @j-ogas: remove ch-docker-run (closes #258) + +If the branch closes multiple issues and it's reasonable to separate those +issues into independent commits, then the branch is rebased, interactively +squashed, and force-pushed into a tidy history with close instructions, then +merged in the web interface with *Create a merge commit*. Example history and +commit messages:: + + * 18aa2b8 merge PR #254 from @j-ogas and me: Dockerfile.openmpi: use snapshot + |\ + | * 79fa89a upgrade to ibverbs 20.0-1 (closes #250) + | * 385ce16 Dockerfile.debian9: use snapshot.debian.org (closes #249) + |/ + * 322df2f ... + +The reason to prefer merge via web interface is that GitHub often doesn't +notice merges done on the command line. + +After merge, the branch is deleted via the web interface. + +**Branch history tidiness.** Commit frequently at semantically relevant times, +and keep in mind that this history will probably be squashed per above. It is +not necessary to rebase or squash to keep branch history tidy. But, don't go +crazy. Commit messages like "try 2" and "fix Travis again" are a bad sign; so +are carefully proofread ones. Commit messages that are brief, technically +relevant, and quick to write are what you want on feature branches. + +**Keep branches up to date.** Merge master into your branch, rather than +rebasing. This lets you resolve conflicts once rather than multiple times as +rebase works through a stack of commits. + +Note that PRs with merge conflicts will generally not be merged. Resolve +conflicts before asking for merge. + +**Remove obsolete branches.** Keep your repo free of old branches with +:code:`git branch -d` (or :code:`-D`) and :code:`git fetch --prune --all`. + +Miscellaneous issue and pull request notes +------------------------------------------ + +**Acknowledging issues.** Issues and PRs submitted from outside should be +acknowledged promptly, including adding or correcting tags. + +**Closing issues.** We close issues when we've taken the requested action, +decided not to take action, resolved the question, or actively determined an +issue is obsolete. It is OK for "stale" issues to sit around indefinitely +awaiting this. Unlike many projects, we do not automatically close issues just +because they're old. - The purpose of this approach is to provide an easy way to see what PRs are - ready to go, without the project lead needing to consult both the list of - PRs and their own list of review requests, and also to provide a way to - request reviews from the project lead without also requesting merge. +**Closing PR.** Stale PRs, on the other hand, are to be avoided due to bit +rot. We try to either merge or reject PRs in a timely manner. - Comments should all be packaged up into a single review; click *Start a - review* rather than *Add single comment*. Then the PR author gets only a - single notification instead of one for every comment you make. +**Re-opening issues.** Closed issues can be re-opened if new information +arises, for example a :code:`worksforme` issue with new reproduction steps. -* Closing issues: We close issues when we've taken the requested action, - decided not to take action, resolved the question, or actively determined an - issue is obsolete. It is OK for "stale" issues to sit around indefinitely - awaiting this. Unlike many projects, we do not automatically close issues - just because they're old. - -* Stale PRs, on the other hand, are to be avoided due to bit rot. We try to - either merge or reject PRs in a timely manner. +Continuous integration testing +------------------------------ -* Closed issues can be re-opened if new information arises, for example a - :code:`worksforme` issue with new reproduction steps. Please comment to ask - for re-opening rather than doing it yourself. +**Quality of testing.** Tagged versions currently get more testing for various +reasons. We are working to improve testing for normal commits on master, but +full parity is probably unlikely. + +**Travis budget.** Because we're on the free tier, we only get 5 Travis jobs +running at a time. Currently, each job takes about ten minutes, there are +seven of them per tested commit, and PRs double this (once on the branch and +once with a test merge commit). The resource is there for your use, so take +advantage of it, but be mindful of the cost, since your fellow developers +might be trying to get in too. + +Things you can do include testing locally first, cancelling jobs you know will +fail or that won't give you additional information, and not pushing every +commit (Travis tests only the most recent commit in a pushed group). + +**Iterating with Travis.** When trying to make Travis happy, use a throwaway +branch that you force-push or squash-merge. Don't submit a PR with half a +dozen "fix Travis" commits. + +**Purging Docker cache.** :code:`test/docker-clean.sh` can be used to purge +your Docker cache, either by removing all tags or deleting all containers and +images. The former is generally preferred, as it lets you update only those +base images that have actually changed (the ones that haven't will be +re-tagged). -GitHub issue and PR tags ------------------------- +GitHub tags +----------- What kind of issue is it? ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -171,7 +262,8 @@ A particularly important or notable issue. :code:`question` - Support request that does not report a problem or ask for a change. + Support request that does not report a problem or ask for a change. Close + these after the question is answered or several days with no activity. What do we plan to do about it? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -197,10 +289,10 @@ caused the "user error".* :code:`ready to merge` - PRs only. Adding this tag states that the PR is complete and requests it be - merged to master. If the project lead requests changes, they'll remove the - tag. Re-add it when you're ready to try again. Lead removes tag after - merging. + PRs only. Adding this tag speculates that the PR is complete and requests it + be considered for merging to master. If the project lead requests changes, + they'll remove the tag. Re-add it when you're ready to try again. Lead + removes tag after merging. :code:`wontfix` We are not going to do this, and we won't merge PRs. Close issue after @@ -211,40 +303,19 @@ We cannot reproduce the issue. Typical workflow is to tag, then wait a few days for clarification before closing. -Testing -------- - -PRs will not be merged until they pass the tests. - -* Tests should pass on your development box as well as all relevant clusters, - in full scope. (Note that some of the examples take quite a long time to - build; the Docker cache is your friend.) - -* All the Travis tests should pass. If you're iterating trying to make Travis - happy, consider interactive rebase, amending commits, or a throwaway branch. - Don't submit a PR with half a dozen "fix Travis" commits. - -* :code:`test/docker-clean.sh` can be used to purge your Docker cache, either - by removing all tags or deleting all containers and images. The former is - generally preferred, as it lets you update only those base images that have - actually changed (the ones that haven't will be re-tagged). +.. _doc-build: Documentation ============= -.. _doc-build: - -How to build the documentation ------------------------------- - This documentation is built using Sphinx with the sphinx-rtd-theme. It lives in :code:`doc-src`. -Prerequisites -~~~~~~~~~~~~~ +Dependencies +------------ - * Python 3.5+ + * Python 3.4+ * Sphinx 1.4.9+ * docutils 0.13.1+ * sphinx-rtd-theme 0.2.4+ @@ -252,9 +323,9 @@ Older versions may work but are untested. To build the HTML -~~~~~~~~~~~~~~~~~ +----------------- -Install the prerequisites:: +Install the dependencies:: $ pip3 install sphinx sphinx-rtd-theme @@ -283,7 +354,7 @@ `_. Publishing to the web -~~~~~~~~~~~~~~~~~~~~~ +--------------------- If you have write access to the repository, you can update the web documentation (i.e., http://hpc.github.io/charliecloud). @@ -355,15 +426,21 @@ Other stuff on the line (e.g., comment syntax) is ignored. +Similarly, you can exclude an architecture with e.g.: + +.. code-block:: none + + ch-test-arch-exclude: aarch64 # ARM not supported upstream + Additional subdirectories can be symlinked into :code:`examples/` and will be integrated into the test suite. This allows you to create a site-specific test suite. -Dockerfile: +:code:`Dockerfile`: * It's a Dockerfile. -Docker_Pull: +:code:`Docker_Pull`: * First line states the address to pull from Docker Hub. * Second line is a scope expression as described above. @@ -374,14 +451,19 @@ alpine:3.6 alpine@sha256:f006ecbb824d87947d0b51ab8488634bf69fe4094959d935c0c103f4820a417d -Build: +:code:`Build`: * Script or program that builds the image. * Arguments: - * :code:`$1`: Absolute path to directory containing Build. - * :code:`$2`: Absolute path and name of gzipped tarball output. + * :code:`$1`: Absolute path to directory containing :code:`Build`. + + * :code:`$2`: Absolute path and name of output archive, without extension. + The script should use an archive format compatible with + :code:`ch-tar2dir` and append the appropriate extension (e.g., + :code:`.tar.gz`). + * :code:`$3`: Absolute path to appropriate temporary directory. * The script must not write anything in the current directory. @@ -397,15 +479,130 @@ tarball with no leading path (:code:`./` is acceptable). * Any programming language is permitted. To be included in the Charliecloud - source code, a language already in the prerequisites is required. + source code, a language already in the test suite dependencies is + required. + + * The script must test for its dependencies and fail with appropriate error + message and exit code if something is missing. To be included in the + Charliecloud source code, all dependencies must be something we are + willing to install and test. * Exit codes: * 0: Image tarball successfully created. - * 65: One or more prerequisites were not met. + * 65: One or more dependencies were not met. + * 126 or 127: No interpreter available for script language (the shell + takes care of this). * else: An error occurred. +Building RPMs +============= + +We maintain :code:`.spec` files and infrastructure for building RPMs in the +Charliecloud source code. This is for two purposes: + + 1. We maintain our own Fedora RPMs (see `packaging guidelines + `_). + + 2. We want to be able to build an RPM of any commit. + +Item 2 is tested; i.e., if you break the RPM build, the test suite will fail. + +This section describes how to build the RPMs and the pain we've hopefully +abstracted away. + +Dependencies +------------ + + * Python 2.7 or 3.4+ + * Either: + + * RPM-based system of roughly RHEL/CentOS 7 vintage or newer, with RPM + build tools installed + * System that can run Charliecloud containers + +:code:`rpmbuild` wrapper script +------------------------------- + +While building the Charliecloud RPMs is not too weird, we provide a script to +streamline it. The purpose is to (a) make it easy to build versions not +matching the working directory, (b) use an arbitrary :code:`rpmbuild` +directory, and (c) build in a Charliecloud container for non-RPM-based +environments. + +The script must be run from the root of a Charliecloud Git working directory. + +Usage:: + + $ packaging/fedora/build [OPTIONS] VERSION + +Options: + + * :code:`--image=DIR` : Build in Charliecloud image directory :code:`DIR`. + + * :code:`--install` : Install the RPMs after building into the build + environment. + + * :code:`--rpmbuild=DIR` : Use RPM build directory root :code:`DIR` + (default: :code:`~/rpmbuild`). + +For example, to build a version 0.9.7 RPM, on an RPM system, and leave the +results in :code:`~/rpmbuild/RPMS`:: + + $ packaging/fedora/build 0.9.7-1 + +To build a pre-release RPM of Git HEAD using the CentOS 7 image provided with +the test suite (note that the test suite would also build the necessary image +directory):: + + $ bin/ch-build -t centos7 -f test/Dockerfile.centos7 test + $ bin/ch-docker2tar centos7 $CH_TEST_TARDIR + $ bin/ch-tar2dir $CH_TEST_TARDIR/centos7.tar.gz $CH_TEST_IMGDIR + $ packaging/fedora/build --image $CH_TEST_IMGDIR/centos7 HEAD + +Gotchas and quirks +------------------ + +RPM versions and releases +~~~~~~~~~~~~~~~~~~~~~~~~~ + +If :code:`VERSION` is :code:`HEAD`, then the RPM version will be the content +of :code:`VERSION.full` for that commit, including Git gobbledygook, and the +RPM release will be :code:`0`. Note that such RPMs cannot be reliably upgraded +because their version numbers are unordered. + +Otherwise, :code:`VERSION` should be a released Charliecloud version followed +by a hyphen and the desired RPM release, e.g. :code:`0.9.7-3`. + +Other values of :code:`VERSION` (e.g., a branch name) may work but are not +supported. + +Packaged source code and RPM build config come from different commits +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The spec file, :code:`build` script, :code:`.rpmlintrc`, etc. come from the +working directory, but the package source is from the specified commit. This +is what enables us to make additional RPM releases for a given Charliecloud +release (e.g. 0.9.7-2). + +Corollaries of this policy are that RPM build configuration can be any or no +commit, and it's not possible to create an RPM of uncommitted source code. + +Changelog maintenance +~~~~~~~~~~~~~~~~~~~~~ + +The spec file contains a manually maintained changelog. Add a new entry for +each new RPM release; do not include the Charliecloud release notes. + +For released versions, :code:`build` verifies that the most recent changelog +entry matches the given :code:`VERSION` argument. The timestamp is not +automatically verified. + +For other Charliecloud versions, :code:`build` adds a generic changelog entry +with the appropriate version stating that it's a pre-release RPM. + + Coding style ============ @@ -440,9 +637,51 @@ * Use spell check. Keep your personal dictionary updated so your editor is not filled with false positives. +.. _dependency-policy: + +Dependency policy +----------------- + +Specific dependencies (prerequisites) are stated elsewhere in the +documentation. This section describes our policy on which dependencies are +acceptable. + +Generally +~~~~~~~~~ + +All dependencies must be stated and justified in the documentation. + +We want Charliecloud to run on as many systems as practical, so we work hard +to keep dependencies minimal. However, because Charliecloud depends on new-ish +kernel features, we do depend on standards of similar vintage. + +Core functionality should be available even on small systems with basic Linux +distributions, so dependencies for run-time and build-time are only the bare +essentials. Exceptions, to be used judiciously: + + * Features that add convenience rather than functionality may have + additional dependencies that are reasonably expected on most systems where + the convenience would be used. + + * Features that only work if some other software is present (example: the + Docker wrapper scripts) can add dependencies of that other software. + +The test suite is tricky, because we need a test framework and to set up +complex test fixtures. We have not yet figured out how to do this at +reasonable expense with dependencies as tight as run- and build-time, so there +are systems that do support Charliecloud but cannot run the test suite. + +Building the documentation needs Sphinx features that have not made their way +into common distributions (i.e., RHEL), so we use recent versions of Sphinx +and provide a source distribution with pre-built documentation. + +Building the RPMs should work on RPM-based distributions with a kernel new +enough to support Charliecloud. You might need to install additional packages +(but not from third-party repositories). + :code:`curl` vs. :code:`wget` ------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For URL downloading in shell code, including Dockerfiles, use :code:`wget -nv`. @@ -486,5 +725,13 @@ "${foo}"/bar # no "$foo"/bar # no +* Don't quote variable assignments or other places where not needed (e.g., + case statements). E.g.: + + .. code-block:: none + + foo=${bar}/baz # yes + foo="${bar}/baz" # no + .. LocalWords: milestoned gh nv cht Chacon's scottchacon diff -Nru charliecloud-0.9.6/doc-src/faq.rst charliecloud-0.9.10/doc-src/faq.rst --- charliecloud-0.9.6/doc-src/faq.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/faq.rst 2019-05-14 20:34:32.000000000 +0000 @@ -116,6 +116,46 @@ Unexpected behavior =================== +What do the version numbers mean? +--------------------------------- + +Released versions of Charliecloud have a pretty standard version number, e.g. +0.9.7. + +Work leading up to a released version also has version numbers, to satisfy +tools that require them and to give the executables something useful to report +on :code:`--version`, but these can be quite messy. We refer to such versions +informally as *pre-releases*, but Charliecloud does not have formal +pre-releases such as alpha, beta, or release candidate. + +*Pre-release version numbers are not in order*, because this work is in a DAG +rather than linear, except they precede the version we are working towards. If +you're dealing with these versions, use Git. + +Pre-release version numbers are the version we are working towards, followed +by: :code:`~pre`, the branch name if not :code:`master` with non-alphanumerics +removed, the commit hash, and finally :code:`dirty` if the working directory +had uncommitted changes. + +Examples: + + * :code:`0.2.0` : Version 0.2.0. Released versions don't include Git + information, even if built in a Git working directory. + + * :code:`0.2.1~pre` : Some snapshot of work leading up to 0.2.1, built from + source code where the Git information has been lost, e.g. the tarballs + Github provides. This should make you wary because you don't have any + provenance. It might even be uncommitted work or an abandoned branch. + + * :code:`0.2.1~pre.1a99f42` : Master branch commit 1a99f42, built from a + clean working directory (i.e., no changes since that commit). + + * :code:`0.2.1~pre.foo1.0729a78` : Commit 0729a78 on branch :code:`foo-1`, + :code:`foo_1`, etc. built from clean working directory. + + * :code:`0.2.1~pre.foo1.0729a78.dirty` : Commit 0729a78 on one of those + branches, plus un-committed changes. + :code:`--uid 0` lets me read files I can’t otherwise! ----------------------------------------------------- @@ -274,6 +314,42 @@ the layers rather than the unpacked image. We do not currently have a fix; see `issue #165 `_. +My second-level directory :code:`dev` is empty +---------------------------------------------- + +Some image tarballs, such as official Ubuntu Docker images, put device files +in :code:`/dev`. These files prevent unpacking the tarball, because +unprivileged users cannot create device files. Further, these files are not +needed because :code:`ch-run` overmounts :code:`/dev` anyway. + +We cannot reliably prevent device files from being included in the tar, +because often that is outside our control, e.g. :code:`docker export` produces +a tarball. Thus, we must exclude them at unpacking time. + +An additional complication is that :code:`ch-tar2dir` can handle tarballs both +with a single top-level directory and without, i.e. “tarbombs”. For example, +best practice use of :code:`tar` on the command line produces the former, +while :code:`docker export` (perhaps via :code:`ch-docker2tar`) produces a +tarbomb. + +Thus, :code:`ch-tar2dir` uses :code:`tar --exclude` to exclude from unpacking +everything under :code:`./dev` and :code:`*/dev`, i.e., directory :code:`dev` +appearing at either the first or second level are forced to be empty. + +This yields false positives if you have a tarbomb image with a directory +:code:`dev` at the second level containing stuff you care about. Hopefully +this is rare, but please let us know if it is your use case. + +My password that contains digits doesn't work in VirtualBox console +------------------------------------------------------------------- + +VirtualBox has confusing Num Lock behavior. Thus, you may be typing arrows, +page up/down, etc. instead of digits, without noticing because console +password fields give no feedback, not even whether a character has been typed. + +Try using the number row instead, toggling Num Lock key, or SSHing into the +virtual machine. + How do I ... ============ @@ -481,65 +557,3 @@ If your X11 application doesn’t work, please file an issue so we can figure out why. - -How do I create a tarball compatible with Charliecloud? -------------------------------------------------------------- - -In contrast with best practices for source code, Charliecloud expects an image -tarball to have either no top-level directory or a top-level directory that is -exactly :code:`.` (dot). This is inherited from the format of :code:`docker -export` tarballs. If you’re creating tarballs by other means, you may run into -this issue. - -For example, let's try to re-pack the :code:`chtest` image directory. This -fails with a rather opaque error message. - -:: - - $ cd $CH_TEST_IMGDIR - $ tar czf chtest2.tar.gz chtest - $ tar tf chtest2.tar.gz | head - chtest/ - chtest/var/ - chtest/var/run/ - chtest/var/empty/ - chtest/var/spool/ - chtest/var/spool/cron/ - chtest/var/spool/cron/crontabs - chtest/var/opt/ - chtest/var/local/ - chtest/var/log/ - $ ch-tar2dir chtest2.tar.gz . - $ ls chtest2 - chtest dev mnt WEIRD_AL_YANKOVIC - $ ch-run ./chtest2 -- echo hello - ch-run[28780]: can't bind /etc/passwd to /var/tmp/images/chtest2/etc/passwd: No such file or directory (charliecloud.c:132 2) - -The workaround is to create the tarball from within the image directory. (If -you do this immediately after the above, you'll need to remove the -:code:`chtest2` directory first.) - -:: - - $ ch $CH_TEST_IMGDIR/chtest - $ tar czf ../chtest2.tar.gz . - $ cd .. - $ tar tf chtest2.tar.gz | head - ./ - ./var/ - ./var/run/ - ./var/empty/ - ./var/spool/ - ./var/spool/cron/ - ./var/spool/cron/crontabs - ./var/opt/ - ./var/local/ - ./var/log/ - $ ch-tar2dir chtest2.tar.gz . - $ ls chtest2 - bin etc lib mnt root sbin sys tmp var - dev home media proc run srv test usr WEIRD_AL_YANKOVIC - $ ch-run ./chtest2 -- echo hello - hello - -We are working on usability enhancements for this process. diff -Nru charliecloud-0.9.6/doc-src/index.rst charliecloud-0.9.10/doc-src/index.rst --- charliecloud-0.9.6/doc-src/index.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/index.rst 2019-05-14 20:34:32.000000000 +0000 @@ -1,6 +1,12 @@ Overview ******** +.. image:: rd100-winner.png + :align: right + :alt: R&D 100 2018 winner logo + :width: 128px + :target: https://www.lanl.gov/discover/news-release-archive/2018/November/1119-rd-100-awards.php + .. include:: ../README.rst .. note:: diff -Nru charliecloud-0.9.6/doc-src/install.rst charliecloud-0.9.10/doc-src/install.rst --- charliecloud-0.9.6/doc-src/install.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/install.rst 2019-05-14 20:34:32.000000000 +0000 @@ -5,78 +5,86 @@ distributions, this can be done using your package manager; otherwise, both normal users and admins can build and install it manually. -.. warning:: - - **If you are installing on a Cray** and have not applied the patch for Cray - case #188073, you must use the `cray branch - `_ to avoid crashing - nodes during job completion. This is a Cray bug that Charliecloud happens - to tickle. There is a patch available from Cray for CLE6.0 UP04 and UP05. - The fix is mainlined into Cray CLE6.0 UP06, released in March 2018. Versions - of Cray CLE6.0 prior to UP04 are unpatched and affected by the bug. - Non-Cray build boxes and others at the same site can still use - the master branch. - .. contents:: :depth: 2 :local: -Prequisites -=========== +Dependencies +============ -Charliecloud is a simple system with limited prerequisites. If your system +Charliecloud is a simple system with limited dependencies. If your system meets these prerequisites but Charliecloud doesn't work, please report that as a bug. +Supported architectures +----------------------- + +Charliecloud should work on any architecture supported by the Linux kernel, +and we have run Charliecloud containers on x86-64, ARM, and Power. However, it +is currently tested only on x86_64 and ARM. + +Most container build software is also fairly portable; e.g., see `Docker's +supported platforms `_. + Run time -------- Systems used for running images need: -* Recent Linux kernel with :code:`CONFIG_USER_NS=y`. We recommend version 4.4 +* Recent Linux kernel with user namespaces enabled. We recommend version 4.4 or higher. -* C compiler and standard library +* C11 compiler and standard library -* POSIX shell and utilities +* POSIX.1-2017 shell and utilities Some distributions need configuration changes to enable user namespaces. For -example, Debian Stretch needs sysctl -:code:`kernel.unprivileged_userns_clone=1`, and RHEL and CentOS 7.4 need both -a `kernel command line option and a sysctl -`_ -(that put you into "technology preview"). +example: + +* Debian Stretch `needs sysctl `_ + :code:`kernel.unprivileged_userns_clone=1`. + +* RHEL/CentOS 7.4 and 7.5 need both a `kernel command line option and a sysctl `_. + *Important note:* Docker does not work with user namespaces, so skip step 4 + of the Red Hat instructions, i.e., don't add :code:`--userns-remap` to the + Docker configuration (see `issue #97 + `_). Build time ---------- -Systems used for building images need the run-time prerequisites, plus: +Systems used for building images need the run-time dependencies, plus +something to actually build the images. -* Bash 4.1+ +A common choice is `Docker `_, along with internet +access or configuration for a local Docker repository. Our wrapper scripts for +Docker expect to run the :code:`docker` command under :code:`sudo` and need +Docker 17.03+ and :code:`mktemp(1)`. (Older versions of Docker may work but +are untested. We know that 1.7.1 does not work.) -and optionally: +Optional build-time dependencies: -* `Docker `_ 17.03+ -* internet access or Docker configured for a local Docker hub -* root access using :code:`sudo` - -Older versions of Docker may work but are untested. We know that 1.7.1 does -not work. +* Bash 4.1+, for :code:`ch-build2dir` Test suite ---------- -In order to run the test suite on a run or build system (you can test each -mode independently), you also need: +To run the test suite, you also need: -* Bash 4.1+ -* Python 2.6+ * `Bats `_ 0.4.0 -* wget - -Note that without Docker on the build system, some of the test suite will be -skipped. +* Bash 4.1+, for Bats and to make programming the tests tractable +* Python 2.7 or 3.4+, for building some of the tests +* Wget, to download stuff for some of the test images +* root access via :code:`sudo` (optional), to test filesystem permissions enforcement + +Image building software tested, with varying levels of thoroughness: + +* Shell scripts with various manual bootstrap and :code:`ch-run` +* Docker +* `Buildah `_ +* `skopeo `_ and + `umoci `_ Bats can be installed at the system level or embedded in the Charliecloud source code. If it's in both places, the latter is used. Binary files /tmp/tmpdhWOHa/CIaYXDaD36/charliecloud-0.9.6/doc-src/logo-sidebar.png and /tmp/tmpdhWOHa/HiRODnG4MA/charliecloud-0.9.10/doc-src/logo-sidebar.png differ Binary files /tmp/tmpdhWOHa/CIaYXDaD36/charliecloud-0.9.6/doc-src/rd100-winner.png and /tmp/tmpdhWOHa/HiRODnG4MA/charliecloud-0.9.10/doc-src/rd100-winner.png differ diff -Nru charliecloud-0.9.6/doc-src/test.rst charliecloud-0.9.10/doc-src/test.rst --- charliecloud-0.9.6/doc-src/test.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/test.rst 2019-05-14 20:34:32.000000000 +0000 @@ -16,10 +16,14 @@ Charliecloud's tests are based in the directory :code:`test`, which is either at the top level of the source code or installed at -:code:`$PREFIX/share/doc/charliecloud`. To run them, go there:: +:code:`$PREFIX/libexec/charliecloud`. To run them, go there:: $ cd test +If you have :code:`sudo`, the tests will make use of it by default. To skip +the tests that use :code:`sudo` even if you have privileges, set +:code:`CH_TEST_DONT_SUDO` to a non-empty string. + The tests use a framework called `Bats `_ (Bash Automated Testing System). To check location and version of Bats used by the tests:: diff -Nru charliecloud-0.9.6/doc-src/vm.rst charliecloud-0.9.10/doc-src/vm.rst --- charliecloud-0.9.6/doc-src/vm.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/doc-src/vm.rst 2019-05-14 20:34:32.000000000 +0000 @@ -75,7 +75,8 @@ the VM "captures" your mouse pointer, type the key combination listed in the lower-right corner of the window to release it.) -4. Change your password: +4. Change your password. (You must use :code:`sudo` because you have + passwordless :code:`sudo` but don't know your password.) :: @@ -233,8 +234,11 @@ Build and provision ------------------- -The most important difference with this build procedure is that a second user -:code:`charlie` is created and endowed with passwordless :code:`sudo`. +The most important differences with this build procedure have to do with +login. A second user :code:`charlie` is created and endowed with passwordless +:code:`sudo`; SSH will allow login with password; and the console will +automatically log in :code:`charlie`. You need to reboot for the latter to +take effect (which is done in the next step). :: @@ -264,9 +268,9 @@ Test Charliecloud ----------------- -:: +Restart and test:: - $ vagrant provision --provision-with=test + $ vagrant up --provision-with=test You might also show the console in the VirtualBox GUI and make sure :code:`charlie` is logged in. diff -Nru charliecloud-0.9.6/examples/mpi/lammps/Dockerfile charliecloud-0.9.10/examples/mpi/lammps/Dockerfile --- charliecloud-0.9.6/examples/mpi/lammps/Dockerfile 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/examples/mpi/lammps/Dockerfile 2019-05-14 20:34:32.000000000 +0000 @@ -1,4 +1,5 @@ # ch-test-scope: full +# ch-test-arch-exclude: aarch64 # issue #392 FROM openmpi WORKDIR / diff -Nru charliecloud-0.9.6/examples/mpi/lammps/test.bats charliecloud-0.9.10/examples/mpi/lammps/test.bats --- charliecloud-0.9.6/examples/mpi/lammps/test.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/examples/mpi/lammps/test.bats 2019-05-14 20:34:32.000000000 +0000 @@ -36,7 +36,8 @@ setup () { scope full - prerequisites_ok lammps + arch_exclude aarch64 # issue #392 + prerequisites_ok "$ch_tag" multiprocess_ok } @@ -54,6 +55,10 @@ } +@test "${ch_tag}/crayify image" { + crayify_mpi_or_skip "$ch_img" +} + @test "${ch_tag}/using all cores" { # shellcheck disable=SC2086 run $ch_mpirun_core ch-run --join "$ch_img" -- \ @@ -84,3 +89,7 @@ # @test "${ch_tag}/python" { skip 'incompatible with --join' lammps_try python; } + +@test "${ch_tag}/revert image" { + unpack_img_all_nodes "$ch_cray" +} diff -Nru charliecloud-0.9.6/examples/mpi/mpibench/test.bats charliecloud-0.9.10/examples/mpi/mpibench/test.bats --- charliecloud-0.9.6/examples/mpi/mpibench/test.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/examples/mpi/mpibench/test.bats 2019-05-14 20:34:32.000000000 +0000 @@ -3,9 +3,6 @@ setup () { scope full prerequisites_ok "$ch_tag" - if [[ $ch_mpi = mpich ]]; then - crayify_mpi_maybe "$ch_img" - fi # - One iteration because we just care about correctness, not performance. # (If we let the benchmark choose, there is an overwhelming number of @@ -38,62 +35,72 @@ # one from "Single Transfer Benchmarks" @test "${ch_tag}/pingpong (guest launch)" { - [[ $ch_cray && $ch_mpi = mpich ]] && skip "issue #255" # shellcheck disable=SC2086 - run ch-run "$ch_img" -- mpirun $ch_mpirun_np "$imb_mpi1" $imb_args PingPong + run ch-run $ch_unslurm "$ch_img" -- \ + mpirun $ch_mpirun_np "$imb_mpi1" $imb_args PingPong echo "$output" [[ $status -eq 0 ]] check_errors "$output" check_process_ct 2 "$output" check_finalized "$output" } -@test "${ch_tag}/pingpong (host launch)" { - multiprocess_ok + +# one from "Parallel Transfer Benchmarks" +@test "${ch_tag}/sendrecv (guest launch)" { # shellcheck disable=SC2086 - run $ch_mpirun_core ch-run --join "$ch_img" -- \ - "$imb_mpi1" $imb_args PingPong + run ch-run $ch_unslurm "$ch_img" -- \ + mpirun $ch_mpirun_np "$imb_mpi1" $imb_args Sendrecv echo "$output" [[ $status -eq 0 ]] check_errors "$output" - check_process_ct 2 "$output" + check_process_ct "$ch_cores_node" "$output" check_finalized "$output" } -# one from "Parallel Transfer Benchmarks" -@test "${ch_tag}/sendrecv (guest launch)" { - [[ $ch_cray && $ch_mpi = mpich ]] && skip "issue #255" +# one from "Collective Benchmarks" +@test "${ch_tag}/allreduce (guest launch)" { # shellcheck disable=SC2086 - run ch-run "$ch_img" -- mpirun $ch_mpirun_np "$imb_mpi1" $imb_args Sendrecv + run ch-run $ch_unslurm "$ch_img" -- \ + mpirun $ch_mpirun_np "$imb_mpi1" $imb_args Allreduce echo "$output" [[ $status -eq 0 ]] check_errors "$output" check_process_ct "$ch_cores_node" "$output" check_finalized "$output" } -@test "${ch_tag}/sendrecv (host launch)" { + +@test "${ch_tag}/crayify image" { + crayify_mpi_or_skip "$ch_img" +} + +@test "${ch_tag}/pingpong (host launch)" { + arch_exclude aarch64 # issue 392 multiprocess_ok # shellcheck disable=SC2086 run $ch_mpirun_core ch-run --join "$ch_img" -- \ - "$imb_mpi1" $imb_args Sendrecv + "$imb_mpi1" $imb_args PingPong echo "$output" [[ $status -eq 0 ]] check_errors "$output" - check_process_ct "$ch_cores_total" "$output" + check_process_ct 2 "$output" check_finalized "$output" } -# one from "Collective Benchmarks" -@test "${ch_tag}/allreduce (guest launch)" { - [[ $ch_cray && $ch_mpi = mpich ]] && skip "issue #255" +@test "${ch_tag}/sendrecv (host launch)" { + arch_exclude aarch64 # issue 392 + multiprocess_ok # shellcheck disable=SC2086 - run ch-run "$ch_img" -- mpirun $ch_mpirun_np "$imb_mpi1" $imb_args Allreduce + run $ch_mpirun_core ch-run --join "$ch_img" -- \ + "$imb_mpi1" $imb_args Sendrecv echo "$output" [[ $status -eq 0 ]] check_errors "$output" - check_process_ct "$ch_cores_node" "$output" + check_process_ct "$ch_cores_total" "$output" check_finalized "$output" } + @test "${ch_tag}/allreduce (host launch)" { + arch_exclude aarch64 # issue 392 multiprocess_ok # shellcheck disable=SC2086 run $ch_mpirun_core ch-run --join "$ch_img" -- \ @@ -104,3 +111,7 @@ check_process_ct "$ch_cores_total" "$output" check_finalized "$output" } + +@test "${ch_tag}/revert image" { + unpack_img_all_nodes "$ch_cray" +} diff -Nru charliecloud-0.9.6/examples/mpi/mpihello/test.bats charliecloud-0.9.10/examples/mpi/mpihello/test.bats --- charliecloud-0.9.6/examples/mpi/mpihello/test.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/examples/mpi/mpihello/test.bats 2019-05-14 20:34:32.000000000 +0000 @@ -3,9 +3,6 @@ setup () { scope full prerequisites_ok "$ch_tag" - if [[ $ch_mpi = mpich ]]; then - crayify_mpi_maybe "$ch_img" - fi } count_ranks () { @@ -15,8 +12,25 @@ | sed -r 's/^.+ ([0-9]+) ranks.+$/\1/' } +@test "${ch_tag}/guest starts ranks" { + # shellcheck disable=SC2086 + run ch-run $ch_unslurm "$ch_img" -- mpirun $ch_mpirun_np /hello/hello + echo "$output" + [[ $status -eq 0 ]] + rank_ct=$(count_ranks "$output") + echo "found ${rank_ct} ranks, expected ${ch_cores_node}" + [[ $rank_ct -eq "$ch_cores_node" ]] + [[ $output = *'0: send/receive ok'* ]] + [[ $output = *'0: finalize ok'* ]] +} + +@test "${ch_tag}/crayify image" { + crayify_mpi_or_skip "$ch_img" +} + @test "${ch_tag}/MPI version" { - run ch-run "$ch_img" -- /hello/hello + # shellcheck disable=SC2086 + run ch-run $ch_unslurm "$ch_img" -- /hello/hello echo "$output" [[ $status -eq 0 ]] if [[ $ch_mpi = openmpi ]]; then @@ -34,28 +48,17 @@ @test "${ch_tag}/serial" { # This seems to start up the MPI infrastructure (daemons, etc.) within the # guest even though there's no mpirun. - run ch-run "$ch_img" -- /hello/hello - echo "$output" - [[ $status -eq 0 ]] - [[ $output = *' 1 ranks'* ]] - [[ $output = *'0: send/receive ok'* ]] - [[ $output = *'0: finalize ok'* ]] -} - -@test "${ch_tag}/guest starts ranks" { - [[ $ch_cray && $ch_mpi = mpich ]] && skip "issue #255" # shellcheck disable=SC2086 - run ch-run "$ch_img" -- mpirun $ch_mpirun_np /hello/hello + run ch-run $ch_unslurm "$ch_img" -- /hello/hello echo "$output" [[ $status -eq 0 ]] - rank_ct=$(count_ranks "$output") - echo "found ${rank_ct} ranks, expected ${ch_cores_node}" - [[ $rank_ct -eq "$ch_cores_node" ]] + [[ $output = *' 1 ranks'* ]] [[ $output = *'0: send/receive ok'* ]] [[ $output = *'0: finalize ok'* ]] } @test "${ch_tag}/host starts ranks" { + arch_exclude aarch64 # issue 392 multiprocess_ok echo "starting ranks with: ${mpirun_core}" @@ -75,8 +78,11 @@ @test "${ch_tag}/Cray bind mounts" { [[ $ch_cray ]] || skip 'host is not a Cray' - [[ $ch_mpi = openmpi ]] && skip 'OpenMPI unsupported on Cray; issue #180' ch-run "$ch_img" -- mount | grep -F /var/opt/cray/alps/spool ch-run "$ch_img" -- mount | grep -F /var/opt/cray/hugetlbfs } + +@test "${ch_tag}/revert image" { + unpack_img_all_nodes "$ch_cray" +} diff -Nru charliecloud-0.9.6/examples/mpi/paraview/Dockerfile charliecloud-0.9.10/examples/mpi/paraview/Dockerfile --- charliecloud-0.9.6/examples/mpi/paraview/Dockerfile 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/examples/mpi/paraview/Dockerfile 2019-05-14 20:34:32.000000000 +0000 @@ -1,4 +1,5 @@ # ch-test-scope: full +# ch-test-arch-exclude: aarch64 # issue #415 FROM openmpi WORKDIR /usr/src @@ -37,7 +38,7 @@ RUN rm -Rf llvm-${LLVM_VERSION}* # Mesa. We need a version newer than Debian provides. -ENV MESA_VERSION 17.3.6 +ENV MESA_VERSION 18.0.5 RUN wget -nv https://mesa.freedesktop.org/archive/mesa-${MESA_VERSION}.tar.xz RUN tar xf mesa-${MESA_VERSION}.tar.xz RUN cd mesa-${MESA_VERSION} \ diff -Nru charliecloud-0.9.6/examples/mpi/paraview/test.bats charliecloud-0.9.10/examples/mpi/paraview/test.bats --- charliecloud-0.9.6/examples/mpi/paraview/test.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/examples/mpi/paraview/test.bats 2019-05-14 20:34:32.000000000 +0000 @@ -2,6 +2,7 @@ setup () { scope full + arch_exclude aarch64 # issue #415 prerequisites_ok paraview indir=$BATS_TEST_DIRNAME outdir=$BATS_TMPDIR @@ -28,8 +29,13 @@ # We do not check .pvtp (and its companion .vtp) output because it's a # collection of XML files containing binary data and it seems too hairy to me. +@test "${ch_tag}/crayify image" { + crayify_mpi_or_skip "$ch_img" +} + @test "${ch_tag}/cone serial" { - ch-run -b "$indir" -b "$outdir" "$ch_img" -- \ + # shellcheck disable=SC2086 + ch-run $ch_unslurm -b "$indir" -b "$outdir" "$ch_img" -- \ pvbatch /mnt/0/cone.py /mnt/1 ls -l "$outdir"/cone* diff -u "${indir}/cone.serial.vtk" "${outdir}/cone.vtk" @@ -55,3 +61,7 @@ diff -u "${indir}/cone.nranks.vtk" "${outdir}/cone.vtk" cmp "${indir}/cone.png" "${outdir}/cone.png" } + +@test "${ch_tag}/revert image" { + unpack_img_all_nodes "$ch_cray" +} diff -Nru charliecloud-0.9.6/.gitmodules charliecloud-0.9.10/.gitmodules --- charliecloud-0.9.6/.gitmodules 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/.gitmodules 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ -[submodule "test/bats"] - path = test/bats - url = https://github.com/sstephenson/bats.git - ignore = untracked diff -Nru charliecloud-0.9.6/Makefile charliecloud-0.9.10/Makefile --- charliecloud-0.9.6/Makefile 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/Makefile 2019-05-14 20:34:32.000000000 +0000 @@ -1,12 +1,15 @@ -SHELL=/bin/bash +SHELL=/bin/sh # Add some good stuff to CFLAGS. -export CFLAGS += -std=c11 -Wall +export CFLAGS += -std=c11 -Wall -g .PHONY: all all: VERSION.full bin/version.h bin/version.sh cd bin && $(MAKE) all - cd test && $(MAKE) all +# only descend into test/ if the right Python is available + if (command -v "$$(head -1 test/make-auto | sed -E 's/^.+ //')"); then \ + cd test && $(MAKE) all; \ + fi cd examples/syscalls && $(MAKE) all .PHONY: clean @@ -16,14 +19,7 @@ cd test && $(MAKE) clean cd examples/syscalls && $(MAKE) clean -# VERSION.full contains the version string reported by the executables. -# -# * If VERSION is an unadorned release (e.g. 0.2.3 not 0.2.3~pre), or there's -# no Git information available, VERSION.full is simply a copy of VERSION. -# -# * Otherwise, we add the Git branch if the current branch is not master, the -# Git commit, and a note if the working directory -# contains uncommitted changes, e.g. "0.2.3~pre+experimental.ae24a4e.dirty". +# VERSION.full contains the version string reported by executables; see FAQ. ifeq ($(shell test -d .git && fgrep -q \~ VERSION && echo true),true) .PHONY: VERSION.full # depends on git metadata, not a simple file VERSION.full: VERSION @@ -31,7 +27,10 @@ (echo "This is a Git working directory but no git found." && false) printf '%s+%s%s%s\n' \ $$(cat $<) \ - $$(git rev-parse --abbrev-ref HEAD | sed 's/.*/&./g' | sed 's/master.//g') \ + $$( git rev-parse --abbrev-ref HEAD \ + | sed 's/[^A-Za-z0-9]//g' \ + | sed 's/$$/./g' \ + | sed 's/master.//g') \ $$(git rev-parse --short HEAD) \ $$(git diff-index --quiet HEAD || echo '.dirty') \ > $@ @@ -44,25 +43,38 @@ bin/version.sh: VERSION.full echo "version () { echo 1>&2 '$$(cat $<)'; }" > $@ -# Yes, this is bonkers. We keep it around even though normal "git archive" or -# the zip files on Github work, because it provides an easy way to create a -# self-contained tarball with embedded Bats and man pages. -# -# You must "cd doc-src && make" before this will work. -.PHONY: export -export: VERSION.full man/charliecloud.1 - test -d .git -a -f test/bats/.git # need recursive Git checkout -# git diff-index --quiet HEAD # need clean working directory +# These targets provide tarballs of HEAD (not the Git working directory) that +# are self-contained, including the source code as well as the man pages +# (both) and Bats (export-bats). To use them in an unclean working directory, +# set $CH_UNCLEAN_EXPORT_OK to non-empty. +# +# You must "cd doc-src && make" before they will work. The targets depend on +# the man pages but don't know how to build them. +# +# They are phony because I haven't figured out their real dependencies. +.PHONY: main.tar +main.tar: VERSION.full man/charliecloud.1 doc/index.html + git diff-index --quiet HEAD || [ -n "$$CH_MAKE_EXPORT_UNCLEAN_OK" ] git archive HEAD --prefix=charliecloud-$$(cat VERSION.full)/ \ -o main.tar + tar --xform=s,^,charliecloud-$$(cat VERSION.full)/, \ + --exclude='.*' \ + -rf main.tar doc man/*.1 VERSION.full + +.PHONY: export +export: main.tar + gzip -9 main.tar + mv main.tar.gz charliecloud-$$(cat VERSION.full).tar.gz + ls -lh charliecloud-$$(cat VERSION.full).tar.gz + +.PHONY: export-bats +export-bats: main.tar + test -d .git -a -f test/bats/.git # need recursive Git checkout cd test/bats && \ git archive HEAD \ --prefix=charliecloud-$$(cat ../../VERSION.full)/test/bats/ \ -o ../../bats.tar tar Af main.tar bats.tar - tar --xform=s,^,charliecloud-$$(cat VERSION.full)/, \ - -rf main.tar \ - man/*.1 VERSION.full gzip -9 main.tar mv main.tar.gz charliecloud-$$(cat VERSION.full).tar.gz rm bats.tar @@ -95,7 +107,6 @@ INSTALL_PREFIX := $(if $(DESTDIR),$(DESTDIR)/$(PREFIX),$(PREFIX)) BIN := $(INSTALL_PREFIX)/bin DOC := $(INSTALL_PREFIX)/share/doc/charliecloud -TEST := $(DOC)/test # LIBEXEC_DIR is modeled after FHS 3.0 and # https://www.gnu.org/prep/standards/html_node/Directory-Variables.html. It # contains any executable helpers that are not needed in PATH. Default is @@ -103,6 +114,7 @@ LIBEXEC_DIR ?= libexec/charliecloud LIBEXEC_INST := $(INSTALL_PREFIX)/$(LIBEXEC_DIR) LIBEXEC_RUN := $(PREFIX)/$(LIBEXEC_DIR) +TEST := $(LIBEXEC_INST)/test .PHONY: install install: all @test -n "$(PREFIX)" || \ @@ -124,34 +136,50 @@ install -d $(INSTALL_PREFIX)/share/man/man1; \ install -pm 644 -t $(INSTALL_PREFIX)/share/man/man1 man/*.1; \ fi -# misc "documentation" +# license and readme install -d $(DOC) install -pm 644 -t $(DOC) LICENSE README.rst +# html files if they were built + if [ -f doc/index.html ]; then \ + cp -r doc $(DOC)/html; \ + rm -f $(DOC)/html/.nojekyll; \ + for i in $$(find $(DOC)/html -type d); do \ + chmod 755 $$i; \ + done; \ + for i in $$(find $(DOC)/html -type f); do \ + chmod 644 $$i; \ + done; \ + fi # examples - for i in examples/syscalls examples/{serial,mpi,other}/*; do \ - install -d $(DOC)/$$i; \ - install -pm 644 -t $(DOC)/$$i $$i/*; \ + for i in examples/syscalls \ + examples/serial/* examples/mpi/* examples/other/*; do \ + install -d $(LIBEXEC_INST)/$$i; \ + install -pm 644 -t $(LIBEXEC_INST)/$$i $$i/*; \ done - chmod 755 $(DOC)/examples/serial/hello/hello.sh \ - $(DOC)/examples/syscalls/pivot_root \ - $(DOC)/examples/syscalls/userns - find $(DOC)/examples -name Build -exec chmod 755 {} \; + chmod 755 $(LIBEXEC_INST)/examples/serial/hello/hello.sh \ + $(LIBEXEC_INST)/examples/syscalls/pivot_root \ + $(LIBEXEC_INST)/examples/syscalls/userns \ + $(LIBEXEC_INST)/examples/*/*/*.sh + find $(LIBEXEC_INST)/examples -name Build -exec chmod 755 {} \; # tests install -d $(TEST) $(TEST)/run install -pm 644 -t $(TEST) test/*.bats test/common.bash test/Makefile install -pm 644 -t $(TEST)/run test/run/*.bats install -pm 755 -t $(TEST) test/Build.* install -pm 644 -t $(TEST) test/Dockerfile.* test/Docker_Pull.* + install -pm 644 -t $(TEST) test/*.patch install -pm 755 -t $(TEST) test/make-auto test/make-perms-test install -d $(TEST)/chtest install -pm 644 -t $(TEST)/chtest test/chtest/* - chmod 755 $(TEST)/chtest/{Build,*.py,printns} - ln -sf ../../../../bin $(TEST)/bin + chmod 755 $(TEST)/chtest/Build \ + $(TEST)/chtest/*.py \ + $(TEST)/chtest/printns + ln -sf ../../../bin $(TEST)/bin # shared library tests install -d $(TEST)/sotest $(TEST)/sotest/bin $(TEST)/sotest/lib - install -pm 755 -t $(TEST)/sotest test/sotest/files_inferrable.txt \ - test/sotest/libsotest.so.1.0 \ - test/sotest/sotest \ + install -pm 755 -t $(TEST)/sotest test/sotest/libsotest.so.1.0 \ + test/sotest/sotest + install -pm 644 -t $(TEST)/sotest test/sotest/files_inferrable.txt \ test/sotest/sotest.c ln -sf ./libsotest.so.1.0 $(TEST)/sotest/libsotest.so ln -sf ./libsotest.so.1.0 $(TEST)/sotest/libsotest.so.1 diff -Nru charliecloud-0.9.6/packaging/fedora/build charliecloud-0.9.10/packaging/fedora/build --- charliecloud-0.9.6/packaging/fedora/build 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/packaging/fedora/build 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,211 @@ +#!/usr/bin/env python2.7 + +# See contributors' guide for documentation of this script. + +from __future__ import print_function + +import argparse +import errno +import os +import pipes +import platform +import pwd +import re +import shutil +import socket +import subprocess +import sys +import time + +CH_BASE = os.path.abspath(os.path.dirname(__file__) + "/../..") +CH_RUN = [CH_BASE + "/bin/ch-run"] +PACKAGES = ["charliecloud", "charliecloud-debuginfo", "charliecloud-test"] +ARCH = platform.machine() + +def main(): + + # Parse arguments. + ap = argparse.ArgumentParser() + ap.add_argument("version") + ap.add_argument("--image", metavar="DIR") + ap.add_argument("--install", action="store_true") + ap.add_argument("--rpmbuild", metavar="DIR", + default="%s/rpmbuild" % os.getenv("HOME")) + args = ap.parse_args() + print("# Charliecloud root: %s" % CH_BASE) + print("""\ +# version: %(version)s +# image: %(image)s +# install: %(install)s +# rpmbuild root: %(rpmbuild)s""" % args.__dict__) + + # What's the real Git version? + if (args.version == "HEAD"): + try: + # If we're on a branch, we want to build on that branch so the branch + # name shows up in the version name. + commit = subprocess.check_output(["git", "symbolic-ref", "-q", + "--short", "HEAD"])[:-1] + except subprocess.CalledProcessError as x: + if (x.returncode != 1): raise + # Detached HEAD (e.g. Travis) is also fine; use commit hash. + commit = subprocess.check_output(["git", "rev-parse", + "--verify", "HEAD"])[:-1] + rpm_release = "0" + else: + m = re.search(r"([0-9.]+)-([0-9]+)", args.version) + commit = "v" + m.group(1) + rpm_release = m.group(2) + + # Create rpmbuild root + rpm_sources = args.rpmbuild + '/SOURCES' + rpm_specs = args.rpmbuild + '/SPECS' + for d in (rpm_sources, rpm_specs): + print("# mkdir -p %s" % d) + try: + os.makedirs(d) + except OSError as x: + if (x.errno != errno.EEXIST): raise + + # Get a clean Git checkout of the desired version. We do this by making a + # temporary clone so as not to mess up the WD. + git_tmp = rpm_sources + '/charliecloud' + print("# cloning into %s and checking out commit %s" % (git_tmp, commit)) + cmd("git", "clone", '.', git_tmp) + cmd("git", "checkout", commit, cwd=git_tmp) + + # Build tarball. + print("# building docs") + cmd("make", "-j2", cwd=git_tmp+"/doc-src") + print("# building source tarball") + cmd("make", "export", cwd=git_tmp) + ch_version = open(git_tmp + "/VERSION.full").read()[:-1] + ch_tarball = "charliecloud-%s.tar.gz" % ch_version + print("# Charliecloud version: %s" % ch_version) + print("# source tarball: %s" % ch_tarball) + os.rename("%s/%s" % (git_tmp, ch_tarball), + "%s/%s" % (rpm_sources, ch_tarball)) + + # Copy lint configuration. + # FIXME: Put version into destination sometime? + shutil.copy("%s/packaging/fedora/charliecloud.rpmlintrc" % CH_BASE, + "%s/charliecloud.rpmlintrc" % rpm_specs) + + # Remove temporary Git directory. + print("# rm -rf %s" % git_tmp) + shutil.rmtree(git_tmp) + + # Copy and patch spec file. + rpm_vr = "%s-%s" % (ch_version, rpm_release) + # Fedora requires no version in spec file. Add a version for pre-release + # specs to make it hard to upload one to Fedora by mistake. + if ("~pre" not in ch_version): + spec = "charliecloud.spec" + else: + spec = "charliecloud-%s.spec" % rpm_vr + with open("%s/packaging/fedora/charliecloud.spec" % CH_BASE, "rt") as in_, \ + open("%s/%s" % (rpm_specs, spec), "wt") as out: + print("# writing %s" % out.name) + t = in_.read() + t = t.replace("@VERSION@", ch_version) + t = t.replace("@RELEASE@", rpm_release) + if ("~pre" in ch_version): + # Add dummy changelog entry. + timestamp = time.strftime("%a %b %d %Y") # required format + name = pwd.getpwuid(os.geteuid()).pw_gecos.split(",")[0] + moniker = pwd.getpwuid(os.geteuid()).pw_name + domain = re.sub(r"^[^.]+.", "", socket.getfqdn()) + t = t.replace("%changelog\n", """\ +%%changelog +* %s %s <%s@%s> %s +- Pre-release package. See Git history for what is going on. +""" % (timestamp, name, moniker, domain, rpm_vr)) + else: + # Verify requested version matches changelog. + m = re.search(r"%changelog\n.+?([0-9.-]+)\n", t) + if (m.group(1) != rpm_vr): + print("requested version %s != changelog %s" % (rpm_vr, m.group(1))) + sys.exit(1) + out.write(t) + + # Prepare build and rpmlint arguments. + container = [] + rpmbuild_args = [] + rpmlint_args = [] + if (not args.image): + rpms = "%s/RPMS/%s" % (args.rpmbuild, ARCH) + rpmbuild_args += ["--define", "_topdir " + args.rpmbuild] + rpmlint_args += ["--file", "%s/charliecloud.rpmlintrc" % rpm_specs] + else: + # Use /usr/local/src because rpmbuild fails with "%{_topdir}/BUILD" + # shorter than "/usr/src/debug" (yes, really!) [1,2]. + # + # [1]: https://access.redhat.com/solutions/1426113 + # [2]: https://gbenson.net/?p=367 + rpms = "/usr/local/src/RPMS/%s" % ARCH + rpm_specs = "/usr/local/src/SPECS" + rpm_sources = "/usr/local/src/SOURCES" + rpmbuild_args += ["--define", "_topdir /usr/local/src"] + rpmlint_args += ["--file", "%s/charliecloud.rpmlintrc" % rpm_specs] + container += [CH_BASE + "/bin/ch-run", "-w", + "-b", "%s:/usr/local/src" % args.rpmbuild, + args.image, "--"] + + # Build RPMs. + cmd(container, "rpmbuild", rpmbuild_args, "--version") + cmd(container, "rpmbuild", rpmbuild_args, "-ba", "%s/%s" % (rpm_specs, spec)) + cmd(container, "ls", "-lh", rpms) + + # Install RPMs. + if (args.install): + print("# uninstalling (most errors can be ignored)") + cmd_ok(container, "rpm", "--erase", PACKAGES) + print("# installing") + for p in PACKAGES: + cmd(container, "rpm", "--install", + "%s/%s-%s.*.rpm" % (rpms, p, rpm_vr)) + cmd(container, "rpm", "-qa", "charliecloud*") + + # Lint RPMs and spec file. Last so problems that don't result in program + # returning error are more obvious. + print("# linting") + cmd(container, "rpmlint", rpmlint_args, "%s/%s" % (rpm_specs, spec)) + for p in PACKAGES: + file_ = "%s/%s-%s.el7.%s.rpm" % (rpms, p, rpm_vr, ARCH) + cmd(container, "test", "-e", file_) + cmd(container, "rpmlint", rpmlint_args, file_) + + # Success! + print("# done") + + +def cmd(*args, **kwargs): + cmd_real(subprocess.check_call, *args, **kwargs) + +def cmd_ok(*args, **kwargs): + rc = cmd_real(subprocess.call, *args, **kwargs) + return (rc == 0) + +def cmd_out(*args, **kwargs): + out = cmd_real(subprocess.check_output, *args, **kwargs) + return out.rstrip() # remove trailing newline + +def cmd_real(runf, *args, **kwargs): + # flatten any sub-lists (kludge) + args2 = [] + for arg in args: + if (isinstance(arg, list)): + args2 += arg + else: + args2.append(arg) + # print and run + print("$", end="") + for arg in args2: + arg = pipes.quote(arg) + print(" " + arg, end="") + print() + return runf(args2, **kwargs) + + +if (__name__ == "__main__"): + main() diff -Nru charliecloud-0.9.6/packaging/fedora/charliecloud.rpmlintrc charliecloud-0.9.10/packaging/fedora/charliecloud.rpmlintrc --- charliecloud-0.9.6/packaging/fedora/charliecloud.rpmlintrc 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/packaging/fedora/charliecloud.rpmlintrc 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,42 @@ +# This file is used to supress false positive errors and warnings generated by +# rpmlint when used with our charliecloud packages. + +# charliecloud.spec + +# The RPM build script will generate invalid source URLs for non-release +# versions, e.g., '0.9.8~pre+epelpackage.41fe9fd'. +addFilter(r'invalid-url') + +# charliecloud + +# Charliecloud uses pivot_root(2), not chroot(2), for containerization. The +# calls to chroot(2) are part of the pivot_root(2) dance and not relevant to +# Charliecloud's security posture. +addFilter(r'missing-call-to-chdir-with-chroot') + +# charliecloud-debuginfo + +# The only files under /usr/lib are those placed there by rpmbuild. +addFilter(r'only-non-binary-in-usr-lib') + +# charliecloud-test + +# Charliecloud is a container runtime. These shared objects are never used in +# the host environment; rather, they are compiled by the test suite (both +# running and examination of which serve as end-user documentation) and injected +# into the container (guest) via utility script 'ch-fromhost'. The ldconfig +# links are generated inside the container runtime environment. For more +# information, see the test file: test/run/ch-fromhost.bats (line 108). +addFilter(r'no-ldconfig-symlink') +addFilter(r'library-without-ldconfig-postin') +addFilter(r'library-without-ldconfig-postun') + +# The test suite has a few C files, e.g. userns.c, pivot_root.c, +# chroot-escape.c, sotest.c, setgroups.c, mknods.c, setuid.c, etc., that +# document -- line-by-line in some cases -- various components of the open source +# runtime. These C files serve to show end users how containers work; some of +# them are used explicitly during test suite runtime. +addFilter(r'devel-file-in-non-devel-package') + +# The symlink to /usr/bin is created and does exist. +addFilter(r'dangling-relative-symlink') diff -Nru charliecloud-0.9.6/packaging/fedora/charliecloud.spec charliecloud-0.9.10/packaging/fedora/charliecloud.spec --- charliecloud-0.9.6/packaging/fedora/charliecloud.spec 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/packaging/fedora/charliecloud.spec 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,105 @@ +# Charliecloud fedora package spec file +# +# Contributors: +# Dave Love @loveshack +# Michael Jennings @mej +# Jordan Ogas @jogas +# Reid Priedhorksy @reidpr + +# Don't try to compile python files with /usr/bin/python +%{?el7:%global __python %__python3} + +# Fedora does not allow SUSE conditionals, thus we define libexecdir to ensure +# consistency. +%define _libexecdir %{_prefix}/libexec + +# Specify python version of a given file +%define versionize_script() (sed -i 's,/env python,/env %1,g' %2) + +%{!?build_cflags:%global build_cflags $RPM_OPT_FLAGS} +%{!?build_ldflags:%global build_ldflags %nil} + +Name: charliecloud +Version: @VERSION@ +Release: @RELEASE@%{?dist} +Summary: Lightweight user-defined software stacks for high-performance computing +License: ASL 2.0 +URL: https://hpc.github.io/%{name}/ +Source0: https://github.com/hpc/%{name}/releases/download/v%{version}/%{name}-%{version}.tar.gz +BuildRequires: gcc + +%package test +Summary: Charliecloud examples and test suite +Requires: %{name}%{?_isa} = %{version}-%{release} +Requires: bats +Requires: bash +Requires: wget +Requires: /usr/bin/python3 + +%description +Charliecloud uses Linux user namespaces to run containers with no privileged +operations or daemons and minimal configuration changes on center resources. +This simple approach avoids most security risks while maintaining access to +the performance and functionality already on offer. + +Container images can be built using Docker or anything else that can generate +a standard Linux filesystem tree. + +For more information: https://hpc.github.io/charliecloud/ + +%description test +Charliecloud test suite and examples. The test suite takes advantage of +container image builders such as Docker, Skopeo, and Buildah. + +%prep +%setup -q + +%{versionize_script python3 test/make-auto} +%{versionize_script python3 test/make-perms-test} + +%build +%make_build CFLAGS="%build_cflags -std=c11 -pthread" LDFLAGS="%build_ldflags" + +%install +%make_install PREFIX=%{_prefix} + +cat > README.EL7 </etc/sysctl.d/51-userns.conf + systemctl -p + +Note for versions below RHEL7.6, you will also need to enable user namespaces: + + grubby --args=namespace.unpriv_enable=1 --update-kernel=ALL + systemctl -p +EOF + +cat > README.TESTS < @VERSION@-@RELEASE@ +- Add initial Fedora/EPEL package. diff -Nru charliecloud-0.9.6/packaging/redhat/charliecloud.spec charliecloud-0.9.10/packaging/redhat/charliecloud.spec --- charliecloud-0.9.6/packaging/redhat/charliecloud.spec 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/packaging/redhat/charliecloud.spec 1970-01-01 00:00:00.000000000 +0000 @@ -1,52 +0,0 @@ -Summary: Lightweight user-defined software stacks for high-performance computing -Name: charliecloud -Version: @VERSION@ -Release: %{?dist} -License: Apache-2.0 -Group: System Environment/Base -URL: https://hpc.github.io/charliecloud/ -Source: %{name}-%{version}.tar.gz -ExclusiveOS: linux -BuildRoot: %{?_tmppath}%{!?_tmppath:/var/tmp}/%{name}-%{version}-%{release}-root -BuildRequires: python python-sphinx python-sphinx_rtd_theme rsync - -%description -Charliecloud provides user-defined software stacks (UDSS) for -high-performance computing (HPC) centers. - -%prep -%setup -q -# Required for CentOS 7 and older, which don't know of Docker lexer yet. -#find doc-src -type f -print0 | xargs -0 sed -i '/.*:language: docker.*/d' - -%build -%{__make} %{?mflags} - -%install -LIBEXEC_POSTFIX=$(echo %{_libexecdir} | sed 's#^/usr/##') -PREFIX=/usr LIBEXEC_DIR=${LIBEXEC_POSTFIX}/charliecloud DESTDIR=$RPM_BUILD_ROOT %{__make} install %{?mflags_install} -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/examples -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/doc -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/test -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/COPYRIGHT -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/LICENSE -rm -rf $RPM_BUILD_ROOT/%{_defaultdocdir}/%{name}/README.rst - -%clean -rm -rf $RPM_BUILD_ROOT - -#%check -#%{__make} -C test test-quick - -%files -%doc LICENSE README.rst examples -%{_mandir}/man1/* - -# Helper scripts -%{_libexecdir}/%{name}/base.sh -%{_libexecdir}/%{name}/version.sh - -# Binaries -%{_bindir}/ch-* - -%changelog diff -Nru charliecloud-0.9.6/packaging/redhat/README charliecloud-0.9.10/packaging/redhat/README --- charliecloud-0.9.6/packaging/redhat/README 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/packaging/redhat/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -You will need to replace @VERSION@ in the .spec file. - -See travis.sh for an example of a build, though under Travis' Ubuntu image. diff -Nru charliecloud-0.9.6/packaging/redhat/travis.sh charliecloud-0.9.10/packaging/redhat/travis.sh --- charliecloud-0.9.6/packaging/redhat/travis.sh 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/packaging/redhat/travis.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,48 +0,0 @@ -#!/bin/bash - -# Build a .rpm from the current source code diretory. $PWD must be the root of -# the Charliecloud source code. -# -# Because this is designed to work on a Ubuntu box, which is Debian-based -# rather than Red Hat, some things are a little odd. - -set -e -set -x - -sudo apt-get install rpm - -RPMBUILD=~/rpmbuild -mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES,SPECS,SRPMS} - -make VERSION.full -VERSION=$(cat VERSION.full) - -tar czf $RPMBUILD/SOURCES/charliecloud-"${VERSION}".tar.gz \ - --xform "s#^\\.#charliecloud-${VERSION}#" \ - --exclude=.git \ - . - -cp packaging/redhat/charliecloud.spec $RPMBUILD/SPECS -sed -i "s#Version: @VERSION@#Version: ${VERSION}#g" $RPMBUILD/SPECS/charliecloud.spec - -# This is handled automatically on Red Hat systems. -sed -i "s#Release: %{?dist}#Release: 1#g" ~/rpmbuild/SPECS/charliecloud.spec - -# Build requirements cannot be satisfied on Debian derivatives. -sed -i 's#BuildRequires:.*##g' ~/rpmbuild/SPECS/charliecloud.spec - -#echo "Prepared rpmbuild directory tree:" -#find $RPMBUILD -#cho "Now starting build!" - -cd $RPMBUILD/SPECS - -rpmbuild -ba charliecloud.spec - -#echo "Done, tree now:" -#find $RPMBUILD - -# RPMLINT not available for 14.04 :-( -#echo "Running rpmlint:" -#cd $RPMBUILD/RPMS -#rpmlint * diff -Nru charliecloud-0.9.6/packaging/vagrant/Vagrantfile charliecloud-0.9.10/packaging/vagrant/Vagrantfile --- charliecloud-0.9.6/packaging/vagrant/Vagrantfile 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/packaging/vagrant/Vagrantfile 2019-05-14 20:34:32.000000000 +0000 @@ -129,7 +129,6 @@ inline: <<-EOF set -e echo 'user.max_user_namespaces = 32767' > /etc/sysctl.d/51-userns.conf - grubby --args='namespace.unpriv_enable=1' --update-kernel=ALL EOF c.vm.provision :reload @@ -162,14 +161,24 @@ env: { "CH_VERSION" => ENV["CH_VERSION"] }, inline: <<-EOF set -e + cd /usr/local/src + sudo chmod 1777 . + + # CentOS/EPEL/IUI don't have the version of shadow-utils (newuidmap and + # newgidmap) we need for runc (used by Buildah), so install from source. + wget -nv https://github.com/shadow-maint/shadow/releases/download/4.6/shadow-4.6.tar.xz + tar xf shadow-4.6.tar.xz + (cd shadow-4.6 && ./configure && sudo make install) + + # Install Buildah. + sudo yum -y install buildah + sudo tee /etc/profile.d/charliecloud.sh << 'EOF2' export CH_TEST_TARDIR=/var/tmp/tarballs export CH_TEST_IMGDIR=/var/tmp/images export CH_TEST_PERMDIRS=skip EOF2 - cd /usr/local/src - sudo chmod 1777 . git clone --recursive https://github.com/hpc/charliecloud.git cd charliecloud @@ -190,6 +199,10 @@ set -e usermod -aG users vagrant echo '%vagrant ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant + + # Configure subuids and subgids for runc. + sudo usermod --add-subuids 10000-65536 vagrant + sudo usermod --add-subgids 10000-65536 vagrant EOF # Remove unneeded packages. @@ -217,6 +230,10 @@ # rebuild, etc. chown -R charlie:charlie /usr/local/src/charliecloud + # Configure subuids and subgids for runc. + sudo usermod --add-subuids 10000-65536 charlie + sudo usermod --add-subgids 10000-65536 charlie + # Automatically log in "charlie" on the console, so they have a way to get # in if SSH isn't working. cd /etc/systemd/system/getty.target.wants @@ -230,6 +247,10 @@ sed -ri 's/^PasswordAuthentication no$/PasswordAuthentication yes/' /etc/ssh/sshd_config systemctl restart sshd + # Fix /etc/shadow permissions. Not clear where they were broken, but + # passwd(1) below fails without this. + sudo restorecon -v /etc/shadow + # Lock out password login for root and vagrant, because the default # password is well-known and we now allow password login. passwd -l root @@ -252,7 +273,7 @@ fi echo "testing as: $user" sudo -iu $user -- sh -c "\ - cd /usr/local/share/doc/charliecloud/test \ + cd /usr/local/libexec/charliecloud/test \ && CH_TEST_SCOPE=$CH_TEST_SCOPE make test" EOF diff -Nru charliecloud-0.9.6/README.rst charliecloud-0.9.10/README.rst --- charliecloud-0.9.6/README.rst 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/README.rst 2019-05-14 20:34:32.000000000 +0000 @@ -22,10 +22,6 @@ * usability and comprehensibility. -**WARNING: Cray CLE in recent versions** has a bug that crashes nodes when -cleaning up after some jobs, including if Charliecloud has been used. See the -installation instructions for a workaround. - How does it work? ----------------- diff -Nru charliecloud-0.9.6/test/bin/charliecloud.c charliecloud-0.9.10/test/bin/charliecloud.c --- charliecloud-0.9.6/test/bin/charliecloud.c 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/bin/charliecloud.c 2019-05-14 20:34:32.000000000 +0000 @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -98,6 +99,7 @@ void join_end(); void log_ids(const char *func, int line); bool path_exists(char *path); +unsigned long path_mount_flags(char *path); void path_split(char *path, char **dir, char **base); void sem_timedwait_relative(sem_t *sem, int timeout); void setup_namespaces(struct container *c); @@ -193,23 +195,24 @@ } // Container /home. if (!c->private_home) { - char *oldhome, *newhome; + char *newhome; // Mount tmpfs on guest /home because guest root is read-only tmpfs_mount("/home", c->newroot, "size=4m"); // Bind-mount user's home directory at /home/$USER. The main use case is // dotfiles. - oldhome = getenv("HOME"); - Tf (oldhome != NULL, "cannot find home directory: $HOME not set"); + Tf (c->old_home != NULL, "cannot find home directory: is $HOME set?"); newhome = cat("/home/", getenv("USER")); Z_ (mkdir(cat(c->newroot, newhome), 0755)); - bind_mount(oldhome, newhome, c->newroot, BD_REQUIRED, 0); + bind_mount(c->old_home, newhome, c->newroot, BD_REQUIRED, 0); } - // Bind-mount /usr/bin/ch-ssh if it exists. - if (path_exists(cat(c->newroot, "/usr/bin/ch-ssh"))) { + // Container /usr/bin/ch-ssh. + if (c->ch_ssh) { char chrun_file[PATH_CHARS]; int len = readlink("/proc/self/exe", chrun_file, PATH_CHARS); T_ (len >= 0); - chrun_file[ lennewroot, "/usr/bin/ch-ssh")), + "--ch-ssh: /usr/bin/ch-ssh not in image"); + chrun_file[ lennewroot, BD_REQUIRED, 0); } @@ -225,7 +228,11 @@ // Re-mount new root read-only unless --write or already read-only. if (!c->writable && !(access(c->newroot, W_OK) == -1 && errno == EROFS)) { - Zf (mount(NULL, c->newroot, NULL, MS_REMOUNT|MS_BIND|MS_RDONLY, NULL), + unsigned long flags = path_mount_flags(c->newroot) + | MS_REMOUNT // Re-mount ... + | MS_BIND // only this mount point ... + | MS_RDONLY; // read-only. + Zf (mount(NULL, c->newroot, NULL, flags, NULL), "can't re-mount image read-only (is it on NFS?)"); } // Pivot into the new root. Use /dev because it's available even in @@ -405,6 +412,38 @@ return false; } +/* Return the mount flags of the file system containing path, suitable for + passing to mount(2). + + This is messy because, the flags we get from statvfs(3) are ST_* while the + flags needed by mount(2) are MS_*. My glibc has a comment in bits/statvfs.h + that the ST_* "should be kept in sync with" the MS_* flags, and the values + do seem to match, but there are additional undocumented flags in there. + Also, the kernel contains a test "unprivileged-remount-test.c" that + manually translates the flags. Thus, I wasn't comfortable simply passing + the output of statvfs(3) to mount(2). */ +unsigned long path_mount_flags(char *path) +{ + struct statvfs sv; + unsigned long known_flags = ST_MANDLOCK | ST_NOATIME | ST_NODEV + | ST_NODIRATIME | ST_NOEXEC | ST_NOSUID + | ST_RDONLY | ST_RELATIME | ST_SYNCHRONOUS; + + Z_ (statvfs(path, &sv)); + Ze (sv.f_flag & ~known_flags, "unknown mount flags: 0x%lx %s", + sv.f_flag & ~known_flags, path); + + return (sv.f_flag & ST_MANDLOCK ? MS_MANDLOCK : 0) + | (sv.f_flag & ST_NOATIME ? MS_NOATIME : 0) + | (sv.f_flag & ST_NODEV ? MS_NODEV : 0) + | (sv.f_flag & ST_NODIRATIME ? MS_NODIRATIME : 0) + | (sv.f_flag & ST_NOEXEC ? MS_NOEXEC : 0) + | (sv.f_flag & ST_NOSUID ? MS_NOSUID : 0) + | (sv.f_flag & ST_RDONLY ? MS_RDONLY : 0) + | (sv.f_flag & ST_RELATIME ? MS_RELATIME : 0) + | (sv.f_flag & ST_SYNCHRONOUS ? MS_SYNCHRONOUS : 0); +} + /* Split path into dirname and basename. */ void path_split(char *path, char **dir, char **base) { @@ -533,6 +572,24 @@ Z_ (unlink(path)); } +/* Split string str at first instance of delimiter del. Set *a to the part + before del, and *b to the part after. Both can be empty; if no token is + present, set both to NULL. Unlike strsep(3), str is unchanged; *a and *b + point into a new buffer allocated with malloc(3). This has two + implications: (1) the caller must free(3) *a but not *b, and (2) the parts + can be rejoined by setting *(*b-1) to del. The point here is to provide an + easier wrapper for strsep(3). */ +void split(char **a, char **b, char *str, char del) +{ + char delstr[2] = { del, 0 }; + T_ (str != NULL); + str = strdup(str); + *b = str; + *a = strsep(b, delstr); + if (*b == NULL) + *a = NULL; +} + /* Mount a tmpfs at the given path. */ void tmpfs_mount(char *dst, char *newroot, char *data) { diff -Nru charliecloud-0.9.6/test/bin/charliecloud.h charliecloud-0.9.10/test/bin/charliecloud.h --- charliecloud-0.9.6/test/bin/charliecloud.h 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/bin/charliecloud.h 2019-05-14 20:34:32.000000000 +0000 @@ -65,6 +65,7 @@ struct container { struct bind *binds; + bool ch_ssh; // bind /usr/bin/ch-ssh? gid_t container_gid; uid_t container_uid; char *newroot; @@ -74,6 +75,7 @@ char *join_tag; // identifier for synchronized join bool private_home; bool private_tmp; + char *old_home; // host path to user's home directory (i.e. $HOME) bool writable; }; @@ -88,4 +90,5 @@ void containerize(struct container *c); void msg(int level, char *file, int line, int errno_, char *fmt, ...); void run_user_command(char *argv[], char *initial_dir); +void split(char **a, char **b, char *str, char del); void version(void); diff -Nru charliecloud-0.9.6/test/bin/ch-build2dir charliecloud-0.9.10/test/bin/ch-build2dir --- charliecloud-0.9.6/test/bin/ch-build2dir 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/bin/ch-build2dir 2019-05-14 20:34:32.000000000 +0000 @@ -1,41 +1,41 @@ -#!/bin/sh +#!/bin/bash libexec="$(cd "$(dirname "$0")" && pwd)" . "${libexec}/base.sh" # shellcheck disable=SC2034 usage=$(cat < "$tar" +docker_ export "$cid" | pv_ -s "$size" > "$tar" docker_ rm "$cid" > /dev/null -ls -lh "$tar" + +# Add the Docker environment variables in ./environment for later consumption +# by "ch-run --set-env". +# +# 1. mktemp(1) isn't POSIX, but it seemed very likely to be installed if +# Docker is, and I couldn't find a more portable way of securely creating +# temporary files. (In particular, I would have preferred to pipe in the +# data rather than creating and deleting a temporary file.) +# +# 2. Blocking factor 1 (-b1) for tar is a bug workaround. Without this switch, +# tar 1.26, which is in RHEL, corrupts the tarball instead of appending to +# it. This doesn't happen in 1.29 in Debian Stretch, and building GNU tar +# from Git source was too hard, so I couldn't bisect a specific commit that +# fixed the bug to learn what exactly was going on. (See PR #371.) +# +# 3. This assumes that the tarball from Docker does not have a single +# top-level directory (i.e., is a tarbomb). +# +echo "adding environment" +temp=$(mktemp --tmpdir ch-docker2tar.XXXXXX) +docker_ inspect "$image" --format='{{range .Config.Env}}{{println .}}{{end}}' \ + > "$temp" +tar rf "$tar" -b1 -P --xform="s|${temp}|environment|" "$temp" +rm "$temp" + +# Finish up. +echo "compressing" +cat "$tar" | pv_ -s "$size" | gzip_ -6 > "${tar}.gz" +rm "$tar" +ls -lh "${tar}.gz" diff -Nru charliecloud-0.9.6/test/bin/ch-fromhost charliecloud-0.9.10/test/bin/ch-fromhost --- charliecloud-0.9.6/test/bin/ch-fromhost 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/bin/ch-fromhost 2019-05-14 20:34:32.000000000 +0000 @@ -73,7 +73,7 @@ ) cray_mpi= # Cray fixups requested -#cray_openmpi= # ... for OpenMPI +cray_openmpi= # ... for OpenMPI cray_mpich= # ... for MPICH dest= image= @@ -228,17 +228,6 @@ [ "$image" ] || fatal "no image specified" -if [ $lib_found ]; then - # We want to put the libraries in the first directory that ldconfig - # searches, so that we can override (or overwrite) any of the same library - # that may already be in the image. - debug "asking ldconfig for shared library destination" - lib_dest=$( "${ch_bin}/ch-run" "$image" -- /sbin/ldconfig -Nv 2> /dev/null \ - | grep -E '^/' | cut -d: -f1 | head -1) - [ -z "${lib_dest%%/*}" ] || fatal "bad path from ldconfig: ${lib_dest}" - debug "shared library destination: ${lib_dest}" -fi - if [ $cray_mpi ]; then sentinel=/etc/opt/cray/release/cle-release [ -f $sentinel ] || fatal "not found: ${sentinel}: are you on a Cray?" @@ -249,8 +238,6 @@ cray_mpich=yes ;; *'Open MPI'*) - # FIXME: remove when implemented - # shellcheck disable=SC2034 cray_openmpi=yes ;; *) @@ -259,6 +246,24 @@ esac fi +if [ $lib_found ]; then + # We want to put the libraries in the first directory that ldconfig + # searches, so that we can override (or overwrite) any of the same library + # that may already be in the image. + debug "asking ldconfig for shared library destination" + # "ldconfig -Nv" gives some pointless warnings on stderr even if + # successful; we don't want to show those to users. However, we don't want + # to simply pipe stderr to /dev/null because this hides real errors. Thus, + # use the following abomination to pipe stdout and stderr to *separate + # grep commands*. See: https://stackoverflow.com/a/31151808 + lib_dest=$( { "${ch_bin}/ch-run" "$image" -- /sbin/ldconfig -Nv \ + 2>&1 1>&3 3>&- | grep -Ev '(^|dynamic linker, ignoring|given more than once)$' ; } \ + 3>&1 1>&2 | grep -E '^/' | cut -d: -f1 | head -1 ) + [ -n "$lib_dest" ] || fatal 'empty path from ldconfig' + [ -z "${lib_dest%%/*}" ] || fatal "bad path from ldconfig: ${lib_dest}" + debug "shared library destination: ${lib_dest}" +fi + if [ $lib_dest_print ]; then echo "$lib_dest" exit 0 @@ -299,21 +304,55 @@ queue_files /opt/cray/alps/default/lib64/libalpslli.so.0.0.0 queue_files /opt/cray/wlm_detect/default/lib64/libwlm_detect.so.0.0.0 #queue_files /opt/cray/alps/default/lib64/libalps.so.0.0.0 +fi - # libwlm_detect.so requires this file to be present. - queue_mkdir /etc/opt/cray/wlm_detect - queue_files /etc/opt/cray/wlm_detect/active_wlm /etc/opt/cray/wlm_detect +if [ $cray_openmpi ]; then + # Both MPI_ROOT and MPIHOME are the base of the OpenMPI install tree. + # We use MPIHOME because some users unset MPI_ROOT. + # + # Note also that the OpenMPI module name is not standardized. + [ "$MPIHOME" ] \ + || fatal "MPIHOME not set; is OpenMPI module loaded?" + + # Inject libmpi from the host + host_libmpi=${MPIHOME}/lib/libmpi.so + [ -f "$host_libmpi" ] \ + || fatal "not found: ${host_libmpi}; is OpenMPI module loaded?" + queue_files "$host_libmpi" + queue_files "$( ldd "$host_libmpi" \ + | grep -E "/usr|/opt" \ + | sed -E 's/^.+ => (.+) \(0x.+\)$/\1/')" + # This appears to be the only dependency in /lib64 that we can't get from + # glibc in the image. + queue_files /lib64/liblustreapi.so + + # Remove libmpi.so* from image. This works around ParaView + # dlopen(3)-related errors that we don't understand. + image_libmpis=$( "${ch_bin}/ch-run" "$image" -- /sbin/ldconfig -p \ + | sed -nr 's|^.* => (/.*/libmpi\.so([0-9.]+)?)$|\1|p') + [ "$image_libmpis" ] || fatal "can't find libmpi.so* in image" + for f in $image_libmpis; do + queue_unlink "$f" + queue_unlink "$("${ch_bin}/ch-run" "$image" -- readlink -f "$f")" + done +fi +if [ $cray_mpi ]; then # ALPS libraries require the contents of this directory to be present at # the same path as the host. Create the mount point here, then ch-run # bind-mounts it later. queue_mkdir /var/opt/cray/alps/spool - # Cray MPICH needs a pile of hugetlbfs filesystems at an arbitrary path - # (it searched /proc/mounts). ch-run bind-mounts to here later. + # libwlm_detect.so requires this file to be present. + queue_mkdir /etc/opt/cray/wlm_detect + queue_files /etc/opt/cray/wlm_detect/active_wlm /etc/opt/cray/wlm_detect + + # uGNI needs a pile of hugetlbfs filesystems at paths that are arbitrary + # but in a specific order in /proc/mounts. ch-run bind-mounts here later. queue_mkdir /var/opt/cray/hugetlbfs fi + [ "$inject_files" ] || fatal "empty file list" debug "injecting into image: ${image}" @@ -343,6 +382,13 @@ [ "$d" ] || fatal "no destination for: ${f}" [ -z "${d%%/*}" ] || fatal "not an absolute path: ${d}" [ -d "${image}${d}" ] || fatal "not a directory: ${image}${d}" + if [ ! -w "${image}${d}" ]; then + # Some images unpack with unwriteable directories; fix. This seems + # like a bit of a kludge to me, so I'd like to remove this special + # case in the future if possible. (#323) + info "${image}${d} not writeable; fixing" + chmod u+w "${image}${d}" || fatal "can't chmod u+w: ${image}${d}" + fi cp --dereference --preserve=all "$f" "${image}${d}" \ || fatal "cannot inject: ${f}" done diff -Nru charliecloud-0.9.6/test/bin/ch-run.c charliecloud-0.9.10/test/bin/ch-run.c --- charliecloud-0.9.6/test/bin/ch-run.c 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/bin/ch-run.c 2019-05-14 20:34:32.000000000 +0000 @@ -6,6 +6,7 @@ #define _GNU_SOURCE #include #include +#include #include #include #include @@ -13,7 +14,6 @@ #include "charliecloud.h" - /** Constants and macros **/ /* Environment variables used by --join parameters. */ @@ -43,28 +43,42 @@ const struct argp_option options[] = { { "bind", 'b', "SRC[:DST]", 0, "mount SRC at guest DST (default /mnt/0, /mnt/1, etc.)"}, - { "cd", 'c', "DIR", 0, "initial working directory in container"}, - { "gid", 'g', "GID", 0, "run as GID within container" }, - { "join", 'j', 0, 0, "use same container as peer ch-run" }, - { "join-pid", -5, "PID", 0, "join a namespace using a PID" }, - { "join-ct", -3, "N", 0, "number of ch-run peers (implies --join)" }, - { "join-tag", -4, "TAG", 0, "label for peer group (implies --join)" }, - { "no-home", -2, 0, 0, "do not bind-mount your home directory"}, - { "private-tmp", 't', 0, 0, "use container-private /tmp" }, - { "uid", 'u', "UID", 0, "run as UID within container" }, - { "verbose", 'v', 0, 0, "be more verbose (debug if repeated)" }, - { "version", 'V', 0, 0, "print version and exit" }, - { "write", 'w', 0, 0, "mount image read-write"}, + { "cd", 'c', "DIR", 0, "initial working directory in container"}, + { "ch-ssh", -8, 0, 0, "bind ch-ssh into image"}, + { "gid", 'g', "GID", 0, "run as GID within container" }, + { "join", 'j', 0, 0, "use same container as peer ch-run" }, + { "join-pid", -5, "PID", 0, "join a namespace using a PID" }, + { "join-ct", -3, "N", 0, "number of ch-run peers (implies --join)" }, + { "join-tag", -4, "TAG", 0, "label for peer group (implies --join)" }, + { "no-home", -2, 0, 0, "do not bind-mount your home directory"}, + { "private-tmp", 't', 0, 0, "use container-private /tmp" }, + { "set-env", -6, "FILE", 0, "set environment variables in FILE"}, + { "uid", 'u', "UID", 0, "run as UID within container" }, + { "unset-env", -7, "GLOB", 0, "unset environment variable(s)" }, + { "verbose", 'v', 0, 0, "be more verbose (debug if repeated)" }, + { "version", 'V', 0, 0, "print version and exit" }, + { "write", 'w', 0, 0, "mount image read-write"}, { 0 } }; +/* One possible future here is that fix_environment() ends up in charliecloud.c + and we add other actions such as SET, APPEND_PATH, etc. */ +enum env_action { END, SET_FILE, UNSET_GLOB }; // END must be zero + +struct env_delta { + enum env_action action; + char *arg; +}; + struct args { struct container c; + struct env_delta *env_deltas; char *initial_dir; }; /** Function prototypes **/ +void env_delta_append(struct env_delta **ds, enum env_action act, char *arg); void fix_environment(struct args *args); bool get_first_env(char **array, char **name, char **value); int join_ct(int cli_ct); @@ -77,12 +91,14 @@ /** Global variables **/ const struct argp argp = { options, parse_opt, args_doc, usage }; +extern char **environ; // see environ(7) /** Main **/ int main(int argc, char *argv[]) { + bool argp_help_fmt_set; struct args args; int arg_next; int c_argc; @@ -91,19 +107,32 @@ privs_verify_invoking(); T_ (args.c.binds = calloc(1, sizeof(struct bind))); + args.c.ch_ssh = false; args.c.container_gid = getegid(); args.c.container_uid = geteuid(); - args.initial_dir = NULL; args.c.join = false; - args.c.join_pid = 0; args.c.join_ct = 0; + args.c.join_pid = 0; args.c.join_tag = NULL; args.c.private_home = false; args.c.private_tmp = false; + args.c.old_home = getenv("HOME"); + args.c.writable = false; + T_ (args.env_deltas = calloc(1, sizeof(struct env_delta))); + args.initial_dir = NULL; verbose = 1; // in charliecloud.h - Z_ (setenv("ARGP_HELP_FMT", "opt-doc-col=25,no-dup-args-note", 0)); + /* I couldn't find a way to set argp help defaults other than this + environment variable. Kludge sets/unsets only if not already set. */ + if (getenv("ARGP_HELP_FMT")) + argp_help_fmt_set = true; + else { + argp_help_fmt_set = false; + Z_ (setenv("ARGP_HELP_FMT", "opt-doc-col=25,no-dup-args-note", 0)); + } Z_ (argp_parse(&argp, argc, argv, 0, &(arg_next), &args)); + if (!argp_help_fmt_set) + Z_ (unsetenv("ARGP_HELP_FMT")); Te (arg_next < argc - 1, "NEWROOT and/or CMD not specified"); args.c.newroot = realpath(argv[arg_next], NULL); @@ -127,8 +156,8 @@ INFO("join: %d %d %s %d", args.c.join, args.c.join_ct, args.c.join_tag, args.c.join_pid); INFO("private /tmp: %d", args.c.private_tmp); - containerize(&args.c); fix_environment(&args); + containerize(&args.c); run_user_command(c_argv, args.initial_dir); // should never return exit(EXIT_FAILURE); } @@ -136,31 +165,112 @@ /** Supporting functions **/ +/* Append a new env_delta to an existing null-terminated list. */ +void env_delta_append(struct env_delta **ds, enum env_action act, char *arg) +{ + int i; + + for (i = 0; (*ds)[i].action != END; i++) // count existing + ; + T_ (*ds = realloc(*ds, (i+2) * sizeof(struct env_delta))); + (*ds)[i+1].action = END; + (*ds)[i].action = act; + (*ds)[i].arg = arg; +} + /* Adjust environment variables. */ void fix_environment(struct args *args) { - char *old, *new; + char *name, *old_value, *new_value; // $HOME: Set to /home/$USER unless --no-home specified. - old = getenv("HOME"); if (!args->c.private_home) { - old = getenv("USER"); - if (old == NULL) { + old_value = getenv("USER"); + if (old_value == NULL) { WARNING("$USER not set; cannot rewrite $HOME"); } else { - T_ (1 <= asprintf(&new, "/home/%s", old)); - Z_ (setenv("HOME", new, 1)); + T_ (1 <= asprintf(&new_value, "/home/%s", old_value)); + Z_ (setenv("HOME", new_value, 1)); } } // $PATH: Append /bin if not already present. - old = getenv("PATH"); - if (old == NULL) { + old_value = getenv("PATH"); + if (old_value == NULL) { WARNING("$PATH not set"); - } else if (strstr(old, "/bin") != old && !strstr(old, ":/bin")) { - T_ (1 <= asprintf(&new, "%s:/bin", old)); - Z_ (setenv("PATH", new, 1)); - INFO("new $PATH: %s", new); + } else if ( strstr(old_value, "/bin") != old_value + && !strstr(old_value, ":/bin")) { + T_ (1 <= asprintf(&new_value, "%s:/bin", old_value)); + Z_ (setenv("PATH", new_value, 1)); + INFO("new $PATH: %s", new_value); + } + + // --set-env and --unset-env. + for (int i = 0; args->env_deltas[i].action != END; i++) { + char *arg = args->env_deltas[i].arg; + if (args->env_deltas[i].action == SET_FILE) { + FILE *fp; + Tf (fp = fopen(arg, "r"), "--set-env: can't open: %s", arg); + for (int j = 1; true; j++) { + char *line = NULL; + size_t len = 0; + errno = 0; + if (-1 == getline(&line, &len, fp)) { + if (errno == 0) // EOF + break; + else // error + Tf (0, "--set-env: error reading: %s", arg); + } + if (strlen(line) == 0 || line[0] == '\n') + continue; // skip empty line + if (line[strlen(line) - 1] == '\n') + line[strlen(line) - 1] = 0; // remove newline + split(&name, &new_value, line, '='); + Te (name != NULL, "--set-env: no delimiter: %s:%d", arg, j); + Te (strlen(name) != 0, "--set-env: empty name: %s:%d", arg, j); + if ( strlen(new_value) >= 2 + && new_value[0] == '\'' + && new_value[strlen(new_value) - 1] == '\'') { + new_value[strlen(new_value) - 1] = 0; // strip trailing quote + new_value++; // strip leading + } + INFO("environment: %s=%s", name, new_value); + Z_ (setenv(name, new_value, 1)); + } + fclose(fp); + } else { + T_ (args->env_deltas[i].action == UNSET_GLOB); + /* Removing variables from the environment is tricky, because there + is no standard library function to iterate through the + environment, and the environ global array can be re-ordered after + unsetenv(3) [1]. Thus, the only safe way without additional + storage is an O(n^2) search until no matches remain. + + It is legal to assign to environ [2]. We build up a copy, omitting + variables that match the glob, which is O(n), and then do so. + + [1]: https://unix.stackexchange.com/a/302987 + [2]: http://man7.org/linux/man-pages/man3/exec.3p.html */ + char **new_environ; + int old_i, new_i; + for (old_i = 0; environ[old_i] != NULL; old_i++) + ; + T_ (new_environ = calloc(old_i + 1, sizeof(char *))); + for (old_i = 0, new_i = 0; environ[old_i] != NULL; old_i++) { + int matchp; + split(&name, &old_value, environ[old_i], '='); + T_ (name != NULL); // env lines should always have equals + matchp = fnmatch(arg, name, 0); + if (!matchp) { + INFO("environment: unset %s", name); + } else { + T_ (matchp == FNM_NOMATCH); + *(old_value - 1) = '='; // rejoin line + new_environ[new_i++] = name; + } + } + environ = new_environ; + } } } @@ -269,6 +379,16 @@ case -5: // --join-pid args->c.join_pid = parse_int(arg, false, "--join-pid"); break; + case -6: // --set-env + env_delta_append(&(args->env_deltas), SET_FILE, arg); + break; + case -7: // --unset-env + Te (strlen(arg) > 0, "--unset-env: GLOB must have non-zero length"); + env_delta_append(&(args->env_deltas), UNSET_GLOB, arg); + break;; + case -8: // --ch-ssh + args->c.ch_ssh = true; + break; case 'c': args->initial_dir = arg; break; diff -Nru charliecloud-0.9.6/test/bin/ch-tar2dir charliecloud-0.9.10/test/bin/ch-tar2dir --- charliecloud-0.9.6/test/bin/ch-tar2dir 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/bin/ch-tar2dir 2019-05-14 20:34:32.000000000 +0000 @@ -1,11 +1,13 @@ #!/bin/sh +set -e + libexec="$(cd "$(dirname "$0")" && pwd)" . "${libexec}/base.sh" # shellcheck disable=SC2034 usage=$(cat <&2 + exit 1 +fi +if [ ! -d "${2}" ]; then + echo "can't unpack: ${2} is not a directory" 1>&2 + exit 1 +fi -# Is the tarball a regular file (or symlink) and readable? -if [ ! -f "$tarball" ] || [ ! -r "$tarball" ]; then - echo "can't read ${tarball}" 1>&2 +# Figure out the real tarball name. If the provided $1 already has a tar +# extension, just test that name; if not, also append the plausible extensions +# and try those too. +for ext in '' .tar.gz .tar.xz .tgz; do + c=${1}${ext} + if [ ! -f "$c" ] || [ ! -r "$c" ]; then + echo "can't read: ${c}" 1>&2 + case $1 in + *.tar.*|*.tgz) + break + ;; + *) + continue + ;; + esac + fi + tarball=$c + if [ -n "$ext" ]; then + echo "found: ${tarball}" 1>&2 + fi + # Infer decompression argument because GNU tar is unable to do so if input + # is a pipe, and we want to keep PV. See: + # https://www.gnu.org/software/tar/manual/html_section/tar_68.html + case $tarball in + *.tar.gz) + newroot=${2}/$(basename "${tarball%.tar.gz}") + decompress=z + ;; + *.tar.xz) + newroot=${2}/$(basename "${tarball%.tar.xz}") + decompress=J + ;; + *.tgz) + newroot=${2}/$(basename "${tarball%.tgz}") + decompress=z + ;; + *) + echo "unknown extension: ${tarball}" 1>&2 + exit 1 + ;; + esac + break +done +if [ -z "$tarball" ]; then + echo "no input found" 1>&2 exit 1 fi @@ -49,18 +97,34 @@ fi mkdir "$newroot" -echo 'This directory is a Charliecloud container image.' > "${newroot}/${sentinel}" # Use a pipe because PV ignores arguments if it's cat rather than PV. +# +# See FAQ on /dev exclusion. --no-wildcards-match-slash is needed to prevent * +# matching multiple directories; the tar default differs from sh behavior. size=$(stat -c%s "$tarball") pv_ -s "$size" < "$tarball" \ -| gzip_ -dc \ -| tar x$verbose -C "$newroot" -f - \ - --anchored --exclude='dev/*' --exclude='./dev/*' +| tar x$decompress -C "$newroot" -f - \ + --anchored --no-wildcards-match-slash \ + --exclude='dev/*' --exclude='*/dev/*' # Make all directories writeable so we can delete image later (hello, Red Hat). find "$newroot" -type d -a ! -perm /200 -exec chmod u+w {} + +# If tarball had a single containing directory, move the contents up a level +# and remove the containing directory. It is non-trivial in POSIX sh to deal +# with hidden files; see https://unix.stackexchange.com/a/6397. +files=$(ls -Aq "$newroot") +if [ "$(echo "$files" | wc -l)" -eq 1 ]; then + ( cd "${newroot}/${files}" + for f in * .[!.]* ..?*; do + if [ -e "$f" ]; then mv -- "$f" ..; fi + done ) + rmdir "${newroot}/${files}" +fi + # Ensure directories that ch-run needs exist. -mkdir -p "${newroot}/dev" +echo 'This directory is a Charliecloud image.' > "${newroot}/${sentinel}" +mkdir -p "${newroot}/dev" "${newroot}/etc" "${newroot}/proc" "${newroot}/sys" +touch "${newroot}/etc/hosts" "${newroot}/etc/resolv.conf" for i in $(seq 0 9); do mkdir -p "${newroot}/mnt/${i}"; done echo "${newroot} unpacked ok" diff -Nru charliecloud-0.9.6/test/build.bats charliecloud-0.9.10/test/build.bats --- charliecloud-0.9.6/test/build.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/build.bats 2019-05-14 20:34:32.000000000 +0000 @@ -58,12 +58,12 @@ for i in "$ch_bin"/ch-*; do echo "shellcheck: ${i}" [[ ! $(file "$i") = *'shell script'* ]] && continue - shellcheck -e SC1090,SC2154 "$i" + shellcheck -e SC1090,SC2002,SC2154 "$i" done # libraries for user executables for i in "$ch_libexec"/*.sh; do echo "shellcheck: ${i}" - shellcheck -s sh -e SC1090 "$i" + shellcheck -s sh -e SC1090,SC2002 "$i" done # BATS scripts # @@ -76,14 +76,14 @@ | shellcheck -s bash -e SC1090,SC2002,SC2154,SC2164 - done < <( find . ../examples -name bats -prune -o -name '*.bats' -print0 ) # libraries for BATS scripts - shellcheck -s bash -e SC2034 ./common.bash + shellcheck -s bash -e SC2002,SC2034 ./common.bash # misc shell scripts if [[ -e ../packaging ]]; then misc=". ../examples ../packaging" else misc=". ../examples" fi - shellcheck -e SC2034 chtest/Build + shellcheck -e SC2002,SC2034 chtest/Build # shellcheck disable=SC2086 while IFS= read -r -d '' i; do echo "shellcheck: ${i}" @@ -121,51 +121,6 @@ [[ $empty_ct -eq 0 ]] } -@test 'ch-build2dir' { - scope standard - # This test unpacks into $ch_tardir so we don't put anything in $ch_imgdir - # at build time. It removes the image on completion. - need_docker - tar="${ch_tardir}/alpine36.tar.gz" - img="${ch_tardir}/test" - [[ ! -e $img ]] - ch-build2dir .. "$ch_tardir" --file=Dockerfile.alpine36 - sudo docker tag test "test:${ch_version_docker}" - docker_ok test - image_ok "$img" - # Remove since we don't want it hanging around later. - rm -Rf --one-file-system "$tar" "$img" -} - -@test 'ch-pull2tar' { - scope standard - # This test pulls an image from Dockerhub and packs it into a tarball at - # $ch_tardir. It removes the tarball upon completetion to keep the number of - # alpine36 tarballs to a minimum. - need_docker - tag='alpine:3.6' - tar="${ch_tardir}/${tag}.tar.gz" - ch-pull2tar "$tag" "$ch_tardir" - [[ $status -eq 0 ]] - [[ -e $tar ]] - rm "${ch_tardir}/${tag}.tar.gz" - [[ ! -e $tar ]] -} - -@test 'ch-pull2dir' { - scope standard - # This test unpacks an image tarball pulled from Docker Hub into $ch_tardir - # to keep $ch_imgdir clean at build time. It removes the image upon completion. - need_docker - tag='alpine:3.6' - img="${ch_tardir}/${tag}" - ch-pull2dir "$tag" "$ch_tardir" - [[ status -eq 0 ]] - [[ -d $img ]] - rm -Rf --one-file-system "$img" - [[ ! -d $img ]] -} - @test 'sotest executable works' { scope quick export LD_LIBRARY_PATH=./sotest diff -Nru charliecloud-0.9.6/test/Build.buildah charliecloud-0.9.10/test/Build.buildah --- charliecloud-0.9.6/test/Build.buildah 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Build.buildah 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,102 @@ +#!/bin/bash +# ch-test-scope: standard + +# Build an Alpine Linux image using Buildah and umoci. This file may be +# removed if/when we can support Buildah instead of Docker for the full test +# suite (#184). +# +# See Build.skopeo-umoci for some caveats also relevant here. +# +# There are three basic approaches we considered: +# +# 1. "buildah export". The main problem here is that the subcommand does not +# exist, though it used to [1,2]. A secondary issue is that it required +# starting up a container [3], which we don't want. +# +# 2. "podman export". Podman (formerly kpod) is a container runtime [4]; it +# can create a container and then export its filesystem. The addition of +# the export command is why "buildah export" was removed [2]. We haven't +# looked into this in detail. It appears to be Buildah's recommended +# approach but would add a dependency. +# +# 3. Export to OCI image with "buildah push", flatten with umoci, and then +# tar up the resulting rootfs. The advantage here is that it uses only +# documented commands; the downside is that it introduces a redundant +# unpack/repack of tarballs +# +# 4. Build with "buildah bud --squash", export to OCI image with "buildah +# push", and manually copy out the tarball of the single layer. The main +# advantage is that we can do it with Buildah only; the disadvantages are +# (1) parsing JSON in shell is a kludge and (2) it only works with +# single-layer images. The latter is easy to get with --squash, but if +# you have a multi-layer image, that will require some fooling around +# that I'm pretty sure is possible but haven't figured out yet. +# +# For now, this script uses approach 4. +# +# [1]: https://github.com/containers/buildah/pull/170 +# [2]: https://github.com/containers/buildah/pull/245 +# [3]: https://github.com/containers/buildah/issues/1118 +# [4]: https://www.projectatomic.io/blog/2018/02/reintroduction-podman + +set -e + +srcdir=$1 +tarball=${2}.tar.gz +workdir=$3 + +tag=alpine39 + +cd "$srcdir" + +if ( ! command -v buildah >/dev/null 2>&1 ); then + echo 'buildah not found' 1>&2 + exit 65 +fi +if ( ! command -v runc >/dev/null 2>&1 ); then + echo 'runc not found' 1>&2 + exit 65 +fi + +# Basic registries.conf file; needed for Buildah to use public Docker Hub. +registries_conf=$(cat <<'EOF' +[registries.search] +registries = ['docker.io'] +EOF +) + +# Build image in Buildah local storage. (Note: This hangs after "Storing +# signatures" if $PWD is not $srcdir.) +export BUILDAH_ISOLATION=rootless +export STORAGE_DRIVER=vfs +buildah --root "${workdir}/storage" --runroot "${workdir}/runroot" \ + --registries-conf <(echo "$registries_conf") \ + build-using-dockerfile \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg no_proxy="$no_proxy" \ + --squash --layers=true -t $tag -f ./Dockerfile.${tag} . + +cd "$workdir" + +# Export an OCI image directory. +buildah --root ./storage --runroot ./runroot push $tag "oci:./oci" + +# Extract the tarball containing the single-layer image from the OCI directory. +manifest=$(sed -E 's|^.+"application/vnd.oci.image.manifest.v1\+json","digest":"sha256:([0-9a-f]+)".+$|\1|' oci/index.json) +echo "manifest: ${manifest}" +layer_ct=$(grep -Eo 'application/vnd.oci.image.layer.v1.tar' \ + "oci/blobs/sha256/${manifest}" | wc -l) +if [[ $layer_ct -ne 1 ]]; then + echo "one layer required; found $layer_ct" 1>&2 + exit 1 +fi +layer=$(sed -E 's|^.+"application/vnd.oci.image.layer.v1.tar","digest":"sha256:([0-9a-f]+)".+$|\1|' "oci/blobs/sha256/${manifest}") +echo "layer: ${layer}" + +# Move the layer tarball to the output (not copy, because the OCI image will +# be deleted when we're done, so OK to break it). +mv "oci/blobs/sha256/${layer}" "$tarball" diff -Nru charliecloud-0.9.6/test/Build.centos7xz charliecloud-0.9.10/test/Build.centos7xz --- charliecloud-0.9.6/test/Build.centos7xz 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Build.centos7xz 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,42 @@ +#!/bin/bash + +# Download an xz-compressed CentOS 7 tarball. These are the base images for +# the official CentOS Docker images. +# +# https://github.com/CentOS/sig-cloud-instance-images +# +# This GitHub repository is arranged with CentOS version and architecture in +# different branches. However, the matrix is not complete: by branch, you can +# download any architecture at the latest version, or a specific version of +# x86_64, but not a specific version of aarch64. Therefore, we download by +# commit hash. +# +# To check what version is in a tarball (on any architecture): +# +# $ tar xf centos-7-${arch}-docker.tar.xz --to-stdout ./etc/centos-release +# +# ch-test-scope: standard + +set -ex + +srcdir=$1 +tarball=${2}.tar.xz +workdir=$3 + +# 7.6.1810 +arch=$(uname -m) +case $arch in + aarch64) + commit=ccc35e0 + ;; + x86_64) + commit=9a389e1 + ;; + *) + echo 'unsupported architecture' 1>&2 + exit 1 +esac + +url="https://github.com/CentOS/sig-cloud-instance-images/blob/${commit}/docker/centos-7-${arch}-docker.tar.xz?raw=true" + +wget -nv -O "$tarball" "$url" diff -Nru charliecloud-0.9.6/test/Build.ch-build2dir charliecloud-0.9.10/test/Build.ch-build2dir --- charliecloud-0.9.6/test/Build.ch-build2dir 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Build.ch-build2dir 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,23 @@ +#!/bin/bash +# ch-test-scope: standard + +# Generate image directory using ch-build2dir and stage it for testing. + +set -e + +srcdir=$1 +tarball_gz=${2}.tar.gz +workdir=$3 + +tag=build2dir + +if ( ! command -v docker &> /dev/null); then + echo 'docker not found' 1>&2 + exit 65 +fi + +cd "$srcdir" +ch-build2dir -t $tag --file=Dockerfile.alpine39 . "$workdir" +cd "$workdir" +tar czf ${tag}.tar.gz $tag +mv ${tag}.tar.gz "$tarball_gz" diff -Nru charliecloud-0.9.6/test/Build.ch-pull2dir charliecloud-0.9.10/test/Build.ch-pull2dir --- charliecloud-0.9.6/test/Build.ch-pull2dir 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Build.ch-pull2dir 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,24 @@ +#!/bin/bash +# ch-test-scope: standard + +# Generate image directory using ch-pull2dir and stage it for testing. + +set -e + +srcdir=$1 +tarball_gz=${2}.tar.gz +workdir=$3 + +imgtag=alpine:3.9 +tag=ch-pull2dir + +if ( ! command -v docker &> /dev/null); then + echo 'docker not found' 1>&2 + exit 65 +fi + +cd "$workdir" +ch-pull2dir "$imgtag" . +mv $imgtag $tag +tar czf ${tag}.tar.gz $tag +mv ${tag}.tar.gz "$tarball_gz" diff -Nru charliecloud-0.9.6/test/Build.ch-pull2tar charliecloud-0.9.10/test/Build.ch-pull2tar --- charliecloud-0.9.6/test/Build.ch-pull2tar 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Build.ch-pull2tar 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,23 @@ +#!/bin/bash +# ch-test-scope: standard + +# Generate a flattened image tarball using ch-pull2tar and stage it for +# testing. + +set -e + +srcdir=$1 +tarball_gz=${2}.tar.gz +workdir=$3 + +imgtag=alpine:3.9 +tag=ch-pull2tar + +if ( ! command -v docker &> /dev/null); then + echo 'docker not found' 1>&2 + exit 65 +fi + +cd "$workdir" +ch-pull2tar $imgtag . +mv ${imgtag}.tar.gz "$tarball_gz" diff -Nru charliecloud-0.9.6/test/build_post.bats charliecloud-0.9.10/test/build_post.bats --- charliecloud-0.9.6/test/build_post.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/build_post.bats 2019-05-14 20:34:32.000000000 +0000 @@ -2,9 +2,8 @@ @test 'nothing unexpected in tarball directory' { scope quick - # We want nothing that's not a .tar.gz or .pg_missing. run find "$ch_tardir" -mindepth 1 \ - -not \( -name '*.tar.gz' -o -name '*.pq_missing' \) + -not \( -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.pq_missing' \) echo "$output" [[ $output = '' ]] } diff -Nru charliecloud-0.9.6/test/Build.skopeo-umoci charliecloud-0.9.10/test/Build.skopeo-umoci --- charliecloud-0.9.6/test/Build.skopeo-umoci 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Build.skopeo-umoci 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,37 @@ +#!/bin/bash + +# Build an Alpine Linux image using skopeo and umoci. This is a precursor to +# proper support within the tools (issue #325). +# +# Note that this approach is a little contrived, in that we unpack the image +# with umoci, tar it up, and then later in the test suite unpack it again +# redundantly with ch-tar2dir. Many real workflows would just use the +# umoci-unpacked image. (umoci does not support producing a tarball directly.) +# +# Warning: This image currently tests the case of a tarball with one top-level +# directory and no hidden files. If you remove it, make sure regressions on +# issue #332 are still tested. +# +# ch-test-scope: standard + +set -ex + +srcdir=$1 +tarball_gz=${2}.tar.gz +workdir=$3 + +cd "$workdir" + +if ( ! command -v skopeo >/dev/null 2>&1 ); then + echo 'skopeo not found' 1>&2 + exit 65 +fi +if ( ! command -v umoci >/dev/null 2>&1 ); then + echo 'umoci not found' 1>&2 + exit 65 +fi + +skopeo copy docker://alpine:3.9 oci:./oci:alpine +umoci unpack --rootless --image ./oci:alpine ./img + +( cd img && tar czf "$tarball_gz" -- rootfs ) diff -Nru charliecloud-0.9.6/test/chtest/Build charliecloud-0.9.10/test/chtest/Build --- charliecloud-0.9.6/test/chtest/Build 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/chtest/Build 2019-05-14 20:34:32.000000000 +0000 @@ -17,18 +17,20 @@ set -ex srcdir=$1 -tarball=$2 -tarball_uncompressed=${tarball%.*} +tarball_uncompressed=${2}.tar +tarball=${tarball_uncompressed}.gz workdir=$3 -mirror=http://dl-cdn.alpinelinux.org/alpine/v3.8 + +arch=$(uname -m) +mirror=http://dl-cdn.alpinelinux.org/alpine/v3.9 # Dynamically select apk-tools-static version. We would prefer to hard-code a # version (and upgrade on our schedule), but we can't because Alpine does not # keep old package versions. If we try, the build breaks every few months (for # example, see issue #242). -apk_tools=$( wget -qO - "${mirror}/main/x86_64" \ +apk_tools=$( wget -qO - "${mirror}/main/${arch}" \ | grep -F apk-tools-static \ | sed -E 's/^.*(apk-tools-static-[0-9.r-]+\.apk).*$/\1/') -img="${workdir}/img" +img=${workdir}/img cd "$workdir" @@ -43,7 +45,7 @@ ## Bootstrap base Alpine Linux. # Download statically linked apk. -wget ${mirror}/main/x86_64/"${apk_tools}" +wget "${mirror}/main/${arch}/${apk_tools}" # Bootstrap directories. mkdir img @@ -51,7 +53,7 @@ touch img/etc/{group,hosts,passwd,resolv.conf} # Bootstrap static apk. -(cd img && tar xf ../"${apk_tools}") +(cd img && tar xf "../${apk_tools}") mkdir img/etc/apk echo ${mirror}/main > img/etc/apk/repositories @@ -96,12 +98,15 @@ # Test programs. cp -r "$srcdir" img/test -$ch_run --cd /test -- make +$ch_run --cd /test -- sh -c 'make clean && make' # Fixtures for /dev cleaning. -touch img/dev/foo +touch img/dev/deleteme mkdir -p img/mnt/dev -touch img/mnt/dev/foo +touch img/mnt/dev/dontdeleteme + +# Fixture to make sure we raise hidden files in non-tarbombs. +touch img/.hiddenfile1 img/..hiddenfile2 img/...hiddenfile3 ## Tar it up. @@ -115,10 +120,17 @@ gzip_cmd=gzip fi -# "docker export" tarballs don't have a leading "./" in their filenames. Match -# that, but also make sure the /dev cleaning fixtures make it in both ways. -cd img -tar cf "$tarball_uncompressed" -- * -tar rf "$tarball_uncompressed" ./dev/foo ./mnt/dev/foo +# Charliecloud supports images both with a single top level directory and +# without (tarbomb). The Docker images in the test suite are all tarbombs +# (because that's what "docker export" gives us), so use a containing +# directory for this one. +tar cf "$tarball_uncompressed" -- img + +# Add in the /dev fixtures in a couple more places. Note that this will cause +# the tarball to no longer have a single root directory, but they'll be +# removed during unpacking, restoring that condition. +( cd img && tar rf "$tarball_uncompressed" ./dev/deleteme dev/deleteme ) + +# Finalize the tarball. $gzip_cmd -f "$tarball_uncompressed" [[ -f $tarball ]] diff -Nru charliecloud-0.9.6/test/chtest/chroot-escape.c charliecloud-0.9.10/test/chtest/chroot-escape.c --- charliecloud-0.9.6/test/chtest/chroot-escape.c 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/chtest/chroot-escape.c 2019-05-14 20:34:32.000000000 +0000 @@ -9,6 +9,7 @@ */ +#define _DEFAULT_SOURCE #include #include #include diff -Nru charliecloud-0.9.6/test/chtest/Makefile charliecloud-0.9.10/test/chtest/Makefile --- charliecloud-0.9.6/test/chtest/Makefile 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/chtest/Makefile 2019-05-14 20:34:32.000000000 +0000 @@ -1,6 +1,6 @@ BINS := chroot-escape mknods setgroups setuid ALL := $(BINS) -CFLAGS := -std=gnu11 -Wall +CFLAGS := -std=c11 -Wall -Werror .PHONY: all all: $(ALL) diff -Nru charliecloud-0.9.6/test/chtest/mknods.c charliecloud-0.9.10/test/chtest/mknods.c --- charliecloud-0.9.6/test/chtest/mknods.c 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/chtest/mknods.c 2019-05-14 20:34:32.000000000 +0000 @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include #include @@ -19,6 +21,7 @@ char * dir; int i, j; unsigned maj, min; + bool open_ok; char * path; for (i = 1; i < argc; i++) { @@ -30,10 +33,10 @@ printf("ERROR\tasprintf() failed with errno=%d\n", errno); return 1; } - fprintf(stderr, "trying to mknod %s\n", path); + fprintf(stderr, "trying to mknod %s: ", path); dev = makedev(maj, min); if (mknod(path, S_IFCHR | 0500, dev)) { - // could not create device, make sure it's an error we expected + // Could not create device; make sure it's an error we expected. switch (errno) { case EACCES: case EINVAL: // e.g. /sys/firmware/efi/efivars @@ -41,21 +44,45 @@ case ENOTDIR: // for bind-mounted files e.g. /etc/passwd case EPERM: case EROFS: + fprintf(stderr, "failed as expected with errno=%d\n", errno); break; default: + fprintf(stderr, "failed with unexpected errno\n"); printf("ERROR\tmknod(2) failed on %s with errno=%d\n", path, errno); return 1; } } else { - // created device OK, now try to remove it + // Device created; safe if we can't open it (see issue #381). + fprintf(stderr, "succeeded\n"); + fprintf(stderr, "trying to open %s: ", path); + if (open(path, O_RDONLY) != -1) { + fprintf(stderr, "succeeded\n"); + open_ok = true; + } else { + open_ok = false; + switch (errno) { + case EACCES: + fprintf(stderr, "failed as expected with errno=%d\n", errno); + break; + default: + fprintf(stderr, "failed with unexpected errno\n"); + printf("ERROR\topen(2) failed on %s with errno=%d\n", + path, errno); + return 1; + } + } + // Remove the device, whether or not we were able to open it. if (unlink(path)) { - printf("ERROR\tmknod(2) succeeded on %s and then unlink(2) " - "failed with errno=%d", path, errno); + printf("ERROR\tunlink(2) failed on %s with errno=%d", + path, errno); + return 1; + } + if (open_ok) { + printf("RISK\tmknod(2), open(2) succeeded on %s (now removed)\n", + path); return 1; } - printf("RISK\tmknod(2) succeeded on %s (now removed)\n", path); - return 1; } } } diff -Nru charliecloud-0.9.6/test/chtest/printns charliecloud-0.9.10/test/chtest/printns --- charliecloud-0.9.6/test/chtest/printns 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/chtest/printns 2019-05-14 20:34:32.000000000 +0000 @@ -1,12 +1,29 @@ -#!/bin/sh +#!/usr/bin/env python3 # Print out my namespace IDs, to stdout or (if specified) the path in $2. -# Then, if $1 is specified, wait that number of seconds before existing. +# Then, if $1 is specified, wait that number of seconds before exiting. -set -e +import glob +import os +import socket +import sys +import time -pause=${1:-0} -out=${2:-/dev/stdout} +if (len(sys.argv) > 1): + pause = float(sys.argv[1]) +else: + pause = 0 -stat -L -c %n:$(hostname -s):%i /proc/self/ns/* > $out -sleep $pause +if (len(sys.argv) > 2): + out = open(sys.argv[2], "wt") +else: + out = sys.stdout + +hostname = socket.gethostname() + +for ns in glob.glob("/proc/self/ns/*"): + stat = os.stat(ns) + print("%s:%s:%d" % (ns, hostname, stat.st_ino), file=out, flush=True) + +if (pause): + time.sleep(pause) diff -Nru charliecloud-0.9.6/test/chtest/setgroups.c charliecloud-0.9.10/test/chtest/setgroups.c --- charliecloud-0.9.6/test/chtest/setgroups.c 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/chtest/setgroups.c 2019-05-14 20:34:32.000000000 +0000 @@ -1,6 +1,7 @@ /* Try to drop the last supplemental group, and print a message to stdout describing what happened. */ +#define _DEFAULT_SOURCE #include #include #include diff -Nru charliecloud-0.9.6/test/common.bash charliecloud-0.9.10/test/common.bash --- charliecloud-0.9.6/test/common.bash 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/common.bash 2019-05-14 20:34:32.000000000 +0000 @@ -1,7 +1,15 @@ -crayify_mpi_maybe () { +arch_exclude () { + if [[ $1 = $(uname -m) ]]; then + skip 'unsupported architecture' + fi +} + +crayify_mpi_or_skip () { if [[ $ch_cray ]]; then # shellcheck disable=SC2086 $ch_mpirun_node ch-fromhost --cray-mpi "$1" + else + skip 'host is not a Cray' fi } @@ -46,11 +54,6 @@ # may not work; we simply haven't tried. [[ $ch_mpi = mpich && -z $ch_cray ]] \ && skip 'MPICH untested' - # Conversely, if the MPI in the container is OpenMPI, the current examples - # do not use the Aries network but rather the "tcp" BTL, which has - # grotesquely poor performance. Thus, we skip those tests as well. - [[ $ch_mpi = openmpi && $ch_cray ]] \ - && skip 'OpenMPI unsupported on Cray; issue #180' # Exit function successfully. true } @@ -104,6 +107,14 @@ test -s "$1" } +unpack_img_all_nodes () { + if [[ $1 ]]; then + $ch_mpirun_node ch-tar2dir "${ch_tardir}/${ch_tag}.tar.gz" "$ch_imgdir" + else + skip 'not needed' + fi +} + # Predictable sorting and collation export LC_ALL=C @@ -210,6 +221,7 @@ else ch_mpirun_np='--use-hwthread-cpus' fi +ch_unslurm= if [[ $SLURM_JOB_ID ]]; then ch_multinode=yes # can run on multiple nodes ch_multiprocess=yes # can run multiple processes @@ -217,6 +229,12 @@ ch_mpirun_core='srun --cpus-per-task 1' # one process/core ch_mpirun_2='srun -n2' # two processes on diff nodes ch_mpirun_2_1node='srun -N1 -n2' # two processes on one node + # OpenMPI 3.1 pukes when guest-launched and Slurm environment variables + # are present. Work around this by fooling OpenMPI into believing it's not + # in a Slurm allocation. + if [[ $ch_mpi = openmpi ]]; then + ch_unslurm='--unset-env=SLURM*' + fi else ch_multinode= if ( command -v mpirun >/dev/null 2>&1 ); then @@ -251,8 +269,9 @@ exit 1 fi -# Do we have sudo? -if ( command -v sudo >/dev/null 2>&1 && sudo -v >/dev/null 2>&1 ); then +# Do we have and want sudo? +if [[ -z $CH_TEST_DONT_SUDO ]] \ + && ( command -v sudo >/dev/null 2>&1 && sudo -v >/dev/null 2>&1 ); then # This isn't super reliable; it returns true if we have *any* sudo # privileges, not specifically to run the commands we want to run. # shellcheck disable=SC2034 diff -Nru charliecloud-0.9.6/test/docker-clean.sh charliecloud-0.9.10/test/docker-clean.sh --- charliecloud-0.9.6/test/docker-clean.sh 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/docker-clean.sh 2019-05-14 20:34:32.000000000 +0000 @@ -14,15 +14,21 @@ sudo docker rm $($cmd) done -# Untag all images -while true; do - cmd='sudo docker images --filter dangling=false --format {{.Repository}}:{{.Tag}}' - tag_ct=$($cmd | wc -l) - echo "found $tag_ct tagged images" - [[ 0 -eq $tag_ct ]] && break - # shellcheck disable=SC2046 - sudo docker rmi -f --no-prune $($cmd) -done +# Untag all images. This fails with: +# +# Error response from daemon: invalid reference format +# +# sometimes. I don't know why. +if [[ $1 != --all ]]; then + while true; do + cmd='sudo docker images --filter dangling=false --format {{.Repository}}:{{.Tag}}' + tag_ct=$($cmd | wc -l) + echo "found $tag_ct tagged images" + [[ 0 -eq $tag_ct ]] && break + # shellcheck disable=SC2046 + sudo docker rmi -f --no-prune $($cmd) + done +fi # If --all specified, remove all images. if [[ $1 = --all ]]; then diff -Nru charliecloud-0.9.6/test/Dockerfile.alpine36 charliecloud-0.9.10/test/Dockerfile.alpine36 --- charliecloud-0.9.6/test/Dockerfile.alpine36 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/Dockerfile.alpine36 1970-01-01 00:00:00.000000000 +0000 @@ -1,7 +0,0 @@ -# ch-test-scope: quick -FROM alpine:3.6 - -RUN apk add --no-cache bc - -# Base image has no default command; we need one to build. -CMD ["true"] diff -Nru charliecloud-0.9.6/test/Dockerfile.alpine39 charliecloud-0.9.10/test/Dockerfile.alpine39 --- charliecloud-0.9.6/test/Dockerfile.alpine39 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Dockerfile.alpine39 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,7 @@ +# ch-test-scope: quick +FROM alpine:3.9 + +RUN apk add --no-cache bc + +# Base image has no default command; we need one to build. +CMD ["true"] diff -Nru charliecloud-0.9.6/test/Dockerfile.centos6 charliecloud-0.9.10/test/Dockerfile.centos6 --- charliecloud-0.9.6/test/Dockerfile.centos6 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/Dockerfile.centos6 2019-05-14 20:34:32.000000000 +0000 @@ -1,4 +1,5 @@ # ch-test-scope: full +# ch-test-arch-exclude: aarch64 # No ARM images provided for CentOS 6 FROM centos:6 RUN yum -y install bc diff -Nru charliecloud-0.9.6/test/Dockerfile.centos7 charliecloud-0.9.10/test/Dockerfile.centos7 --- charliecloud-0.9.6/test/Dockerfile.centos7 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/Dockerfile.centos7 2019-05-14 20:34:32.000000000 +0000 @@ -1,5 +1,16 @@ -# ch-test-scope: full +# ch-test-scope: standard FROM centos:7 -RUN yum -y install bc +# This image has two purposes: (1) demonstrate we can build a CentOS 7 image +# and (2) provide a build environment for Charliecloud RPMs. + +RUN yum -y install epel-release +RUN yum -y install \ + bats \ + gcc \ + make \ + python36 \ + rpm-build \ + rpmlint \ + wget RUN yum clean all diff -Nru charliecloud-0.9.6/test/Dockerfile.debian9 charliecloud-0.9.10/test/Dockerfile.debian9 --- charliecloud-0.9.6/test/Dockerfile.debian9 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/Dockerfile.debian9 2019-05-14 20:34:32.000000000 +0000 @@ -1,6 +1,7 @@ # ch-test-scope: standard FROM debian:stretch +ENV chse_dockerfile foo ENV DEBIAN_FRONTEND noninteractive RUN apt-get update \ diff -Nru charliecloud-0.9.6/test/Dockerfile.nvidia charliecloud-0.9.10/test/Dockerfile.nvidia --- charliecloud-0.9.6/test/Dockerfile.nvidia 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/Dockerfile.nvidia 2019-05-14 20:34:32.000000000 +0000 @@ -1,4 +1,5 @@ # ch-test-scope: full +# ch-test-arch-exclude: aarch64 # only x86-64 and ppc64le supported by nVidia # This Dockerfile demonstrates a multi-stage build. With a single-stage build # that brings along the nVidia build environment, the resulting unpacked image diff -Nru charliecloud-0.9.6/test/Dockerfile.openmpi charliecloud-0.9.10/test/Dockerfile.openmpi --- charliecloud-0.9.6/test/Dockerfile.openmpi 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/Dockerfile.openmpi 2019-05-14 20:34:32.000000000 +0000 @@ -49,10 +49,11 @@ # 2. --disable-pty-support to avoid "pipe function call failed when # setting up I/O forwarding subsystem". # -# 3. --with-slurm=no so mpirun in the container doesn't try use Slurm to -# launch processes, which will fail in inscrutable ways e.g. if srun is -# not found. (If you do want Slurm launching processes, use srun or -# mpirun outside the container.) +# 3. --enable-mca-no-build=plm-slurm to support launching processes using +# the host's srun (i.e., the container OpenMPI needs to talk to the host +# Slurm's PMI) but prevent OpenMPI from invoking srun itself from within +# the container, where srun is not installed (the error messages from +# this are inscrutable). # # [1]: https://github.com/open-mpi/ompi/blob/master/README @@ -71,6 +72,7 @@ libhwloc-dev \ libnl-3-200 \ libnl-route-3-200 \ + libnl-route-3-dev \ libnuma1 \ libpmi2-0-dev \ make \ @@ -83,12 +85,17 @@ # (too old in Stretch). Download manually because I'm too lazy to set up # package pinning. # -# Note that libpsm2 is x86_64 only: -# https://lists.debian.org/debian-hpc/2017/12/msg00015.html -ENV DEB_URL http://snapshot.debian.org/archive/debian/20180928T210119Z/pool/main -ENV PSM2_VERSION 10.3.58-2 -RUN wget -nv ${DEB_URL}/libp/libpsm2/libpsm2-2_${PSM2_VERSION}_amd64.deb -RUN wget -nv ${DEB_URL}/libp/libpsm2/libpsm2-dev_${PSM2_VERSION}_amd64.deb +# Note that libpsm2 is x86-64 only: +# https://packages.debian.org/buster/libpsm2-2 +# https://lists.debian.org/debian-hpc/2017/12/msg00015.html +ENV DEB_URL http://snapshot.debian.org/archive/debian/20181126T030749Z/pool/main +ENV PSM2_VERSION 11.2.68-3 +RUN [ "$(dpkg --print-architecture)" = "amd64" ] \ + || wget -nv ${DEB_URL}/libp/libpsm2/libpsm2-2_${PSM2_VERSION}_amd64.deb \ + ${DEB_URL}/libp/libpsm2/libpsm2-dev_${PSM2_VERSION}_amd64.deb + +# As of 5/2/2019, this is not the newest libibverbs. However, it is the +# newest that doesn't crash on our test systems. ENV IBVERBS_VERSION 20.0-1 RUN for i in ibacm \ ibverbs-providers \ @@ -102,9 +109,8 @@ rdma-core \ rdmacm-utils ; \ do \ - wget -nv ${DEB_URL}/r/rdma-core/${i}_${IBVERBS_VERSION}_amd64.deb ; \ + wget -nv ${DEB_URL}/r/rdma-core/${i}_${IBVERBS_VERSION}_$(dpkg --print-architecture).deb ; \ done -RUN dpkg --install *.deb # UCX. There is stuff to build Debian packages, but it seems not too polished. ENV UCX_VERSION 1.3.1 @@ -115,24 +121,30 @@ && ./contrib/configure-release --prefix=/usr/local \ && make -j$(getconf _NPROCESSORS_ONLN) install +# Install the .debs we collected. +RUN dpkg --install *.deb + # OpenMPI. -ENV MPI_URL https://www.open-mpi.org/software/ompi/v2.1/downloads -ENV MPI_VERSION 2.1.5 +# +# Patch OpenMPI to disable UCX plugin on systems with Intel or Cray HSNs. UCX +# has inferior performance than PSM2/uGNI but higher priority. +ENV MPI_URL https://www.open-mpi.org/software/ompi/v3.1/downloads +ENV MPI_VERSION 3.1.4 RUN wget -nv ${MPI_URL}/openmpi-${MPI_VERSION}.tar.gz RUN tar xf openmpi-${MPI_VERSION}.tar.gz +COPY dont-init-ucx-on-intel-cray.patch ./openmpi-${MPI_VERSION} +RUN cd openmpi-${MPI_VERSION} && git apply dont-init-ucx-on-intel-cray.patch RUN cd openmpi-${MPI_VERSION} \ && CFLAGS=-O3 \ CXXFLAGS=-O3 \ ./configure --prefix=/usr/local \ --sysconfdir=/mnt/0 \ + --with-slurm \ --with-pmi \ - --with-pmi-libdir=/usr/lib/x86_64-linux-gnu \ --with-pmix \ - --with-psm2 \ --with-ucx \ --disable-pty-support \ - --enable-mca-no-build=btl-openib \ - --with-slurm=no \ + --enable-mca-no-build=btl-openib,plm-slurm \ && make -j$(getconf _NPROCESSORS_ONLN) install RUN ldconfig RUN rm -Rf openmpi-${MPI_VERSION}* diff -Nru charliecloud-0.9.6/test/Docker_Pull.alpine36_dp charliecloud-0.9.10/test/Docker_Pull.alpine36_dp --- charliecloud-0.9.6/test/Docker_Pull.alpine36_dp 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/Docker_Pull.alpine36_dp 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -alpine:3.6 -ch-test-scope: quick diff -Nru charliecloud-0.9.6/test/Docker_Pull.alpine39_dp charliecloud-0.9.10/test/Docker_Pull.alpine39_dp --- charliecloud-0.9.6/test/Docker_Pull.alpine39_dp 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/Docker_Pull.alpine39_dp 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,2 @@ +alpine:3.9 +ch-test-scope: quick diff -Nru charliecloud-0.9.6/test/dont-init-ucx-on-intel-cray.patch charliecloud-0.9.10/test/dont-init-ucx-on-intel-cray.patch --- charliecloud-0.9.6/test/dont-init-ucx-on-intel-cray.patch 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/dont-init-ucx-on-intel-cray.patch 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,26 @@ +diff --git a/ompi/mca/pml/ucx/pml_ucx_component.c b/ompi/mca/pml/ucx/pml_ucx_component.c +index ff0040f18c..e8cf903860 100644 +--- a/ompi/mca/pml/ucx/pml_ucx_component.c ++++ b/ompi/mca/pml/ucx/pml_ucx_component.c +@@ -14,6 +14,9 @@ + + #include + ++#ifdef HAVE_UNISTD_H ++#include ++#endif + + static int mca_pml_ucx_component_register(void); + static int mca_pml_ucx_component_open(void); +@@ -131,6 +134,11 @@ mca_pml_ucx_component_init(int* priority, bool enable_progress_threads, + { + int ret; + ++ if ((0 == access("/sys/class/ugni/", F_OK) || (0 == access("/sys/class/hfi1/", F_OK)))){ ++ PML_UCX_VERBOSE(1, "Cray or Intel HSN detected, removing UCX from consideration"); ++ return NULL; ++ } ++ + if ( (ret = mca_pml_ucx_init()) != 0) { + return NULL; + } diff -Nru charliecloud-0.9.6/test/make-auto charliecloud-0.9.10/test/make-auto --- charliecloud-0.9.6/test/make-auto 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/make-auto 2019-05-14 20:34:32.000000000 +0000 @@ -41,13 +41,18 @@ if (dirname == ""): dirname = "." tag = tag_from_path(path) - # Find scope + # Interpret test hints. with open(path) as fp: - m = re.search(r"ch-test-scope: (skip|quick|standard|full)", fp.read()) + text = fp.read() + # ch-test-scope + m = re.search(r"ch-test-scope: (skip|quick|standard|full)", text) if (m is None): print("%s: no valid scope specified" % path, file=sys.stderr) sys.exit(1) scope = m.group(1) + # ch-test-arch-exclude + arch_exclude = "\n".join("arch_exclude %s" % i for i in + re.findall(r"ch-test-arch-exclude: (\w+)", text)) # Build a tarball: different test for each type. if (mode == "build"): @@ -55,7 +60,8 @@ template = """\ @test 'custom build %(tag)s' { scope %(scope)s - tarball=$ch_tardir/%(tag)s.tar.gz + %(arch_exclude)s + tarball=$ch_tardir/%(tag)s pq=$ch_tardir/%(tag)s.pq_missing workdir=$ch_tardir/%(tag)s.tmp rm -f "$pq" @@ -75,6 +81,7 @@ template = """\ @test 'ch-build %(tag)s' { scope %(scope)s + %(arch_exclude)s need_docker %(tag)s ch-build -t %(tag)s --file="%(path)s" "%(dirname)s" sudo docker tag %(tag)s "%(tag)s:$ch_version_docker" @@ -87,6 +94,7 @@ template = """\ @test 'docker pull %(tag)s' { scope %(scope)s + %(arch_exclude)s need_docker %(tag)s sudo docker pull %(addr)s sudo docker tag %(addr)s %(tag)s @@ -96,9 +104,11 @@ template += """ @test 'ch-docker2tar %(tag)s' { scope %(scope)s + %(arch_exclude)s need_docker tarball="$ch_tardir/%(tag)s.tar.gz" ch-docker2tar %(tag)s "$ch_tardir" + tar -tf "$tarball" | grep -E '^environment$' tarball_ok "$tarball" }""" else: @@ -111,11 +121,13 @@ print("""\ @test 'ch-tar2dir %(tag)s' { scope %(scope)s + %(arch_exclude)s prerequisites_ok %(tag)s - ch-tar2dir "$ch_tardir/%(tag)s.tar.gz" "$ch_imgdir" + ch-tar2dir "$ch_tardir/%(tag)s" "$ch_imgdir" } @test 'ch-run %(tag)s /bin/true' { scope %(scope)s + %(arch_exclude)s prerequisites_ok %(tag)s img="$ch_imgdir/%(tag)s" ch-run "$img" /bin/true diff -Nru charliecloud-0.9.6/test/README.md charliecloud-0.9.10/test/README.md --- charliecloud-0.9.6/test/README.md 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/README.md 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,2 @@ +Charliecloud comes with a fairly comprehensive Bats test suite. For testing +instructions visit: https://hpc.github.io/charliecloud/test.html diff -Nru charliecloud-0.9.6/test/run/build-rpms.bats charliecloud-0.9.10/test/run/build-rpms.bats --- charliecloud-0.9.6/test/run/build-rpms.bats 1970-01-01 00:00:00.000000000 +0000 +++ charliecloud-0.9.10/test/run/build-rpms.bats 2019-05-14 20:34:32.000000000 +0000 @@ -0,0 +1,54 @@ +load ../common + +@test 'build/install/uninstall RPMs' { + scope standard + prerequisites_ok centos7 + [[ -d ../.git ]] || skip "not in Git working directory" + command -v sphinx-build > /dev/null 2>&1 || skip 'Sphinx is not installed' + img=${ch_imgdir}/centos7 + + # Build and install RPMs into CentOS 7 image. + (cd .. && packaging/fedora/build --install --image="$img" \ + --rpmbuild="$BATS_TMPDIR/rpmbuild" HEAD) + + # Do installed RPMs look sane? + run ch-run "$img" -- rpm -qa "charliecloud*" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'charliecloud-'* ]] + [[ $output = *'charliecloud-debuginfo-'* ]] + [[ $output = *'charliecloud-test-'* ]] + run ch-run "$img" -- rpm -ql "charliecloud" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'/usr/bin/ch-run'* ]] + [[ $output = *'/usr/libexec/charliecloud/base.sh'* ]] + [[ $output = *'/usr/share/man/man1/charliecloud.1.gz'* ]] + run ch-run "$img" -- rpm -ql "charliecloud-debuginfo" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'/usr/lib/debug/usr/bin/ch-run.debug'* ]] + [[ $output = *'/usr/lib/debug/usr/libexec/charliecloud/test/sotest/lib/libsotest.so.1.0.debug'* ]] + run ch-run "$img" -- rpm -ql "charliecloud-test" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'/usr/libexec/charliecloud/examples/mpi/lammps/Dockerfile'* ]] + [[ $output = *'/usr/libexec/charliecloud/test/Build.centos7xz'* ]] + [[ $output = *'/usr/libexec/charliecloud/test/sotest/lib/libsotest.so.1.0'* ]] + + # Uninstall to avoid interfering with the rest of the test suite. + run ch-run -w "$img" -- rpm -v --erase charliecloud-test \ + charliecloud-debuginfo \ + charliecloud + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'charliecloud-'* ]] + [[ $output = *'charliecloud-debuginfo-'* ]] + [[ $output = *'charliecloud-test-'* ]] + + # All gone? + run ch-run "$img" -- rpm -qa "charliecloud*" + echo "$output" + [[ $status -eq 0 ]] + [[ $output = '' ]] +} diff -Nru charliecloud-0.9.6/test/run/ch-fromhost.bats charliecloud-0.9.10/test/run/ch-fromhost.bats --- charliecloud-0.9.6/test/run/ch-fromhost.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/run/ch-fromhost.bats 2019-05-14 20:34:32.000000000 +0000 @@ -2,12 +2,17 @@ fromhost_clean () { [[ $1 ]] - rm -f "$1"/{lib,mnt,usr/bin}/sotest \ - "$1"/{lib,mnt,usr/lib,usr/local/lib}/libsotest.so.1{.0,} \ - "$1"/mnt/sotest.c \ - "$1"/etc/ld.so.cache \ - "$1"/usr/local/lib/libcuda* \ - "$1"/usr/local/lib/libnvidia* + # We used to delete only specific paths, but this turned into an unwieldy + # mess of wildcards that obscured the original specificity purpose. + rm -f "${1}/ld.so.cache" + find "$1" -xdev \( \ + -name 'libcuda*' \ + -o -name 'libnvidia*' \ + -o -name libsotest.so.1 \ + -o -name libsotest.so.1.0 \ + -o -name sotest \ + -o -name sotest.c \ + \) -print -delete ch-run -w "$1" -- /sbin/ldconfig # restore default cache fromhost_clean_p "$1" } @@ -29,28 +34,27 @@ prerequisites_ok debian9 img=${ch_imgdir}/debian9 - # inferred path is what we expect - [[ $(ch-fromhost --lib-path "$img") = /usr/local/lib ]] + libpath=$(ch-fromhost --lib-path "$img") + echo "libpath: ${libpath}" # --file fromhost_clean "$img" ch-fromhost -v --file sotest/files_inferrable.txt "$img" fromhost_ls "$img" test -f "${img}/usr/bin/sotest" - test -f "${img}/usr/local/lib/libsotest.so.1.0" - test -L "${img}/usr/local/lib/libsotest.so.1" + test -f "${img}${libpath}/libsotest.so.1.0" + test -L "${img}${libpath}/libsotest.so.1" ch-run "$img" -- /sbin/ldconfig -p | grep -F libsotest ch-run "$img" -- sotest rm "${img}/usr/bin/sotest" - rm "${img}/usr/local/lib/libsotest.so.1.0" - rm "${img}/usr/local/lib/libsotest.so.1" + rm "${img}${libpath}/libsotest.so.1.0" + rm "${img}${libpath}/libsotest.so.1" ch-run -w "$img" -- /sbin/ldconfig fromhost_clean_p "$img" # --cmd ch-fromhost -v --cmd 'cat sotest/files_inferrable.txt' "$img" ch-run "$img" -- sotest - fromhost_clean "$img" # --path ch-fromhost -v --path sotest/bin/sotest \ @@ -82,9 +86,9 @@ # --no-ldconfig ch-fromhost -v --no-ldconfig --file sotest/files_inferrable.txt "$img" - test -f "${img}/usr/bin/sotest" - test -f "${img}/usr/local/lib/libsotest.so.1.0" - ! test -L "${img}/usr/local/lib/libsotest.so.1" + test -f "${img}/usr/bin/sotest" + test -f "${img}${libpath}/libsotest.so.1.0" + ! test -L "${img}${libpath}/libsotest.so.1" ! ( ch-run "$img" -- /sbin/ldconfig -p | grep -F libsotest ) run ch-run "$img" -- sotest echo "$output" @@ -96,6 +100,13 @@ ch-fromhost --file sotest/files_inferrable.txt "$img" ch-run "$img" -- sotest fromhost_clean "$img" + + # destination directory not writeable (#323) + chmod -v u-w "${img}/mnt" + ch-fromhost --dest /mnt --path sotest/sotest.c "$img" + test -w "${img}/mnt" + test -f "${img}/mnt/sotest.c" + fromhost_clean "$img" } @test 'ch-fromhost (CentOS)' { @@ -261,6 +272,19 @@ [[ $status -eq 1 ]] [[ $output = *'duplicate image'* ]] fromhost_clean_p "$img" + + # ldconfig gives no shared library path (#324) + # + # (I don't think this is the best way to get ldconfig to fail, but I + # couldn't come up with anything better. E.g., bad ld.so.conf or broken + # .so's seem to produce only warnings.) + mv "${img}/sbin/ldconfig" "${img}/sbin/ldconfig.foo" + run ch-fromhost --lib-path "$img" + mv "${img}/sbin/ldconfig.foo" "${img}/sbin/ldconfig" + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *'empty path from ldconfig'* ]] + fromhost_clean_p "$img" } @test 'ch-fromhost --cray-mpi not on a Cray' { diff -Nru charliecloud-0.9.6/test/run/ch-run_isolation.bats charliecloud-0.9.10/test/run/ch-run_isolation.bats --- charliecloud-0.9.6/test/run/ch-run_isolation.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/run/ch-run_isolation.bats 2019-05-14 20:34:32.000000000 +0000 @@ -26,7 +26,7 @@ | grep -Em1 '[A-Za-z] [0-9]' \ | sed -r 's/^(.*")?(.+)(")$/\2/') echo "host: ${host_distro}" - guest_expected='Alpine Linux v3.8' + guest_expected='Alpine Linux v3.9' echo "guest expected: ${guest_expected}" if [[ $host_distro = "$guest_expected" ]]; then skip 'host matches expected guest distro' diff -Nru charliecloud-0.9.6/test/run/ch-run_join.bats charliecloud-0.9.10/test/run/ch-run_join.bats --- charliecloud-0.9.6/test/run/ch-run_join.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/run/ch-run_join.bats 2019-05-14 20:34:32.000000000 +0000 @@ -200,6 +200,7 @@ ch-run -v --join-ct=3 --join-tag=foo "$ch_timg" -- \ /test/printns 0 "${BATS_TMPDIR}/join.3.ns" \ >& "${BATS_TMPDIR}/join.3.err" & + sleep 1 cat "${BATS_TMPDIR}/join.3.err" cat "${BATS_TMPDIR}/join.3.ns" grep -Fq 'join: 1 3' "${BATS_TMPDIR}/join.3.err" diff -Nru charliecloud-0.9.6/test/run/ch-run_misc.bats charliecloud-0.9.10/test/run/ch-run_misc.bats --- charliecloud-0.9.6/test/run/ch-run_misc.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/run/ch-run_misc.bats 2019-05-14 20:34:32.000000000 +0000 @@ -1,16 +1,19 @@ load ../common + @test 'relative path to image' { # issue #6 scope quick cd "$(dirname "$ch_timg")" && ch-run "$(basename "$ch_timg")" -- true } + @test 'symlink to image' { # issue #50 scope quick ln -sf "$ch_timg" "${BATS_TMPDIR}/symlink-test" ch-run "${BATS_TMPDIR}/symlink-test" -- true } + @test 'mount image read-only' { scope quick run ch-run "$ch_timg" sh < write' ch-run -w "$ch_timg" rm write } + @test '/usr/bin/ch-ssh' { + # Note: --ch-ssh without /usr/bin/ch-ssh is in test "broken image errors". scope quick ls -l "$ch_bin/ch-ssh" - ch-run "$ch_timg" -- ls -l /usr/bin/ch-ssh - ch-run "$ch_timg" -- test -x /usr/bin/ch-ssh + ch-run --ch-ssh "$ch_timg" -- ls -l /usr/bin/ch-ssh + ch-run --ch-ssh "$ch_timg" -- test -x /usr/bin/ch-ssh + # Test bind-mount by comparing size rather than e.g. "ch-ssh --version" + # because ch-ssh won't run on Alpine (issue #4). host_size=$(stat -c %s "${ch_bin}/ch-ssh") - guest_size=$(ch-run "$ch_timg" -- stat -c %s /usr/bin/ch-ssh) + guest_size=$(ch-run --ch-ssh "$ch_timg" -- stat -c %s /usr/bin/ch-ssh) echo "host: ${host_size}, guest: ${guest_size}" [[ $host_size -eq "$guest_size" ]] } + @test 'optional default bind mounts silently skipped' { scope standard @@ -50,6 +59,7 @@ ch-run "$ch_timg" -- mount | ( ! grep -F /var/opt/cray/hugetlbfs ) } + # shellcheck disable=SC2016 @test '$HOME' { scope quick @@ -80,7 +90,7 @@ echo "$output" [[ $status -eq 1 ]] # shellcheck disable=SC2016 - [[ $output = *'cannot find home directory: $HOME not set'* ]] + [[ $output = *'cannot find home directory: is $HOME set?'* ]] # warn if $USER not set user_tmp=$USER @@ -95,6 +105,7 @@ [[ $output = *"$HOME"* ]] } + # shellcheck disable=SC2016 @test '$PATH: add /bin' { scope quick @@ -124,6 +135,7 @@ [[ $output = $PATH2:/bin ]] } + # shellcheck disable=SC2016 @test '$PATH: unset' { scope standard @@ -139,6 +151,7 @@ [[ $output = *'True'* ]] } + @test 'ch-run --cd' { scope quick # Default initial working directory is /. @@ -160,6 +173,7 @@ [[ $output =~ "can't cd to /goops: No such file or directory" ]] } + @test 'ch-run --bind' { scope quick # one bind, default destination (/mnt/0) @@ -200,6 +214,7 @@ ch-run --no-home -b "${ch_imgdir}/bind1" "$ch_timg" -- cat /mnt/0/file1 } + @test 'ch-run --bind errors' { scope quick @@ -277,6 +292,269 @@ [[ $output = *"can't bind: not found: ${ch_timg}/goops"* ]] } + +@test 'ch-run --set-env' { + scope standard + + # Quirk that is probably too obscure to put in the documentation: The + # string containing only two straight quotes does not round-trip through + # "printenv" or "env", though it does round-trip through Bash "set": + # + # $ export foo="''" + # $ echo [$foo] + # [''] + # $ set | fgrep foo + # foo=''\'''\''' + # $ eval $(set | fgrep foo) + # $ echo [$foo] + # [''] + # $ printenv | fgrep foo + # foo='' + # $ eval $(printenv | fgrep foo) + # $ echo $foo + # [] + + # Valid inputs. Use Python to print the results to avoid ambiguity. + f_in=${BATS_TMPDIR}/env.txt + cat <<'EOF' > "$f_in" +chse_a1=bar +chse_a2=bar=baz +chse_a3=bar baz +chse_a4='bar' +chse_a5= +chse_a6='' +chse_a7='''' + +chse_b1="bar" +chse_b2=bar # baz +chse_b3=$PATH + chse_b4=bar +chse_b5= bar + +chse_c1=foo +chse_c1=bar +EOF + cat "$f_in" + output_expected=$(cat <<'EOF' +(' chse_b4', 'bar') +('chse_a1', 'bar') +('chse_a2', 'bar=baz') +('chse_a3', 'bar baz') +('chse_a4', 'bar') +('chse_a5', '') +('chse_a6', '') +('chse_a7', "''") +('chse_b1', '"bar"') +('chse_b2', 'bar # baz') +('chse_b3', '$PATH') +('chse_b5', ' bar') +('chse_c1', 'bar') +EOF +) + run ch-run --set-env="$f_in" "$ch_timg" -- python3 -c 'import os; [print((k,v)) for (k,v) in sorted(os.environ.items()) if "chse_" in k]' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") +} + +@test 'ch-run --set-env from Dockerfile' { + scope standard + prerequisites_ok debian9 + img=${ch_imgdir}/debian9 + + output_expected=$(cat <<'EOF' +chse_dockerfile=foo +EOF +) + + run ch-run --set-env="${img}/environment" "$img" -- \ + sh -c 'env | grep -E "^chse_"' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") +} + +@test 'ch-run --set-env errors' { + scope standard + f_in=${BATS_TMPDIR}/env.txt + + # file does not exist + run ch-run --set-env=doesnotexist.txt "$ch_timg" -- /bin/true + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"--set-env: can't open:"* ]] + [[ $output = *"No such file or directory"* ]] + + # Note: I'm not sure how to test an error during reading, i.e., getline(3) + # rather than fopen(3). Hence no test for "error reading". + + # invalid line: missing '=' + echo 'FOO bar' > "$f_in" + run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"--set-env: no delimiter: ${f_in}:1"* ]] + + # invalid line: no name + echo '=bar' > "$f_in" + run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"--set-env: empty name: ${f_in}:1"* ]] +} + +@test 'ch-run --unset-env' { + scope standard + + export chue_1=foo + export chue_2=bar + + printf '\n# Nothing\n\n' + run ch-run --unset-env=doesnotmatch "$ch_timg" -- env + echo "$output" + [[ $status -eq 0 ]] + ex='^(_|HOME|PATH)=' # variables expected to change + diff -u <(env | grep -Ev "$ex") <(echo "$output" | grep -Ev "$ex") + + printf '\n# Everything\n\n' + run ch-run --unset-env='*' "$ch_timg" -- env + echo "$output" + [[ $status -eq 0 ]] + [[ $output = '' ]] + + printf '\n# Everything, plus shell re-adds\n\n' + run ch-run --unset-env='*' "$ch_timg" -- /bin/sh -c env + echo "$output" + [[ $status -eq 0 ]] + diff -u <(printf 'SHLVL=1\nPWD=/\n') <(echo "$output") + + printf '\n# Without wildcards\n\n' + run ch-run --unset-env=chue_1 "$ch_timg" -- env + echo "$output" + [[ $status -eq 0 ]] + diff -u <(printf 'chue_2=bar\n') <(echo "$output" | grep -E '^chue_') + + printf '\n# With wildcards\n\n' + run ch-run --unset-env='chue_*' "$ch_timg" -- env + echo "$output" + [[ $status -eq 0 ]] + [[ $(echo "$output" | grep -E '^chue_') = '' ]] + + printf '\n# Empty string\n\n' + run ch-run --unset-env= "$ch_timg" -- env + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *'--unset-env: GLOB must have non-zero length'* ]] +} + +@test 'ch-run mixed --set-env and --unset-env' { + scope standard + + # Input. + export chmix_a1=z + export chmix_a2=y + export chmix_a3=x + f1_in=${BATS_TMPDIR}/env1.txt + cat <<'EOF' > "$f1_in" +chmix_b1=w +chmix_b2=v +EOF + f2_in=${BATS_TMPDIR}/env2.txt + cat <<'EOF' > "$f2_in" +chmix_c1=u +chmix_c2=t +EOF + + # unset, unset + output_expected=$(cat <<'EOF' +chmix_a3=x +EOF +) + run ch-run --unset-env=chmix_a1 --unset-env=chmix_a2 "$ch_timg" -- \ + sh -c 'env | grep -E ^chmix_ | sort' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") + + echo '# set, set' + output_expected=$(cat <<'EOF' +chmix_a1=z +chmix_a2=y +chmix_a3=x +chmix_b1=w +chmix_b2=v +chmix_c1=u +chmix_c2=t +EOF +) + run ch-run --set-env="$f1_in" --set-env="$f2_in" "$ch_timg" -- \ + sh -c 'env | grep -E ^chmix_ | sort' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") + + echo '# unset, set' + output_expected=$(cat <<'EOF' +chmix_a2=y +chmix_a3=x +chmix_b1=w +chmix_b2=v +EOF +) + run ch-run --unset-env=chmix_a1 --set-env="$f1_in" "$ch_timg" -- \ + sh -c 'env | grep -E ^chmix_ | sort' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") + + echo '# set, unset' + output_expected=$(cat <<'EOF' +chmix_a1=z +chmix_a2=y +chmix_a3=x +chmix_b1=w +EOF +) + run ch-run --set-env="$f1_in" --unset-env=chmix_b2 "$ch_timg" -- \ + sh -c 'env | grep -E ^chmix_ | sort' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") + + echo '# unset, set, unset' + output_expected=$(cat <<'EOF' +chmix_a2=y +chmix_a3=x +chmix_b1=w +EOF +) + run ch-run --unset-env=chmix_a1 \ + --set-env="$f1_in" \ + --unset-env=chmix_b2 \ + "$ch_timg" -- sh -c 'env | grep -E ^chmix_ | sort' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") + + echo '# set, unset, set' + output_expected=$(cat <<'EOF' +chmix_a1=z +chmix_a2=y +chmix_a3=x +chmix_b1=w +chmix_c1=u +chmix_c2=t +EOF +) + run ch-run --set-env="$f1_in" \ + --unset-env=chmix_b2 \ + --set-env="$f2_in" \ + "$ch_timg" -- sh -c 'env | grep -E ^chmix_ | sort' + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$output_expected") <(echo "$output") +} + @test 'broken image errors' { scope standard img="${BATS_TMPDIR}/broken-image" @@ -285,7 +563,7 @@ dirs=$(echo {dev,proc,sys}) files=$(echo etc/{group,hosts,passwd,resolv.conf}) # shellcheck disable=SC2116 - files_optional=$(echo usr/bin/ch-ssh) + files_optional= # formerly for ch-ssh (#378), but leave infrastructure mkdir -p "$img" for d in $dirs; do mkdir -p "${img}/$d"; done mkdir -p "${img}/etc" "${img}/home" "${img}/usr/bin" "${img}/tmp" @@ -394,9 +672,23 @@ [[ $status -eq 1 ]] [[ $output = *"can't execve(2): true: No such file or directory"* ]] + # --ch-ssh but no /usr/bin/ch-ssh + run ch-run --ch-ssh "$img" -- true + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"--ch-ssh: /usr/bin/ch-ssh not in image"* ]] + # Everything should be restored and back to the original error. run ch-run "$img" -- true echo "$output" [[ $status -eq 1 ]] [[ $output = *"can't execve(2): true: No such file or directory"* ]] + + # At this point, there should be exactly two each of passwd and group + # temporary files. Remove them. + [[ $(find /tmp -maxdepth 1 -name 'ch-run_passwd*' | wc -l) -eq 2 ]] + [[ $(find /tmp -maxdepth 1 -name 'ch-run_group*' | wc -l) -eq 2 ]] + rm -v /tmp/ch-run_{passwd,group}* + [[ $(find /tmp -maxdepth 1 -name 'ch-run_passwd*' | wc -l) -eq 0 ]] + [[ $(find /tmp -maxdepth 1 -name 'ch-run_group*' | wc -l) -eq 0 ]] } diff -Nru charliecloud-0.9.6/test/run/ch-run_uidgid.bats charliecloud-0.9.10/test/run/ch-run_uidgid.bats --- charliecloud-0.9.6/test/run/ch-run_uidgid.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/run/ch-run_uidgid.bats 2019-05-14 20:34:32.000000000 +0000 @@ -103,8 +103,9 @@ printf 'RISK\tsuccessful mount\n' return 1 ;; - 1) ;& # "incorrect invocation of permissions" (we care which) - 255) # undocumented + 1) ;& # "incorrect invocation or permissions" (we care which) + 111) ;& # undocumented + 255) # undocumented if [[ $output = *'ermission denied'* ]]; then printf 'SAFE\tmount exit %d, permission denied\n' "$status" return 0 diff -Nru charliecloud-0.9.6/test/run/ch-tar2dir.bats charliecloud-0.9.10/test/run/ch-tar2dir.bats --- charliecloud-0.9.6/test/run/ch-tar2dir.bats 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/run/ch-tar2dir.bats 2019-05-14 20:34:32.000000000 +0000 @@ -10,22 +10,68 @@ image_ok "$ch_timg" ch-tar2dir "$ch_ttar" "$ch_imgdir" # overwrite image_ok "$ch_timg" + # Did we raise hidden files correctly? + [[ -e $ch_timg/.hiddenfile1 ]] + [[ -e $ch_timg/..hiddenfile2 ]] + [[ -e $ch_timg/...hiddenfile3 ]] } @test 'ch-tar2dir: /dev cleaning' { # issue #157 scope standard - [[ ! -e $ch_timg/dev/foo ]] - [[ -e $ch_timg/mnt/dev/foo ]] - ch-run "$ch_timg" -- test -e /mnt/dev/foo + # Are all fixtures present in tarball? + present=$(tar tf "$ch_ttar" | grep -F deleteme) + [[ $(echo "$present" | wc -l) -eq 4 ]] + echo "$present" | grep -E '^img/dev/deleteme$' + echo "$present" | grep -E '^./dev/deleteme$' + echo "$present" | grep -E '^dev/deleteme$' + echo "$present" | grep -E '^img/mnt/dev/dontdeleteme$' + # Did we remove the right fixtures? + [[ -e $ch_timg/mnt/dev/dontdeleteme ]] + [[ $(ls -Aq "${ch_timg}/dev") -eq 0 ]] + ch-run "$ch_timg" -- test -e /mnt/dev/dontdeleteme } @test 'ch-tar2dir: errors' { scope quick - # tarball doesn't exist + + # destination doesn't exist + run ch-tar2dir "$ch_timg" /doesnotexist + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"can't unpack: /doesnotexist does not exist"* ]] + + # destination is not a directory + run ch-tar2dir "$ch_timg" /bin/false + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"can't unpack: /bin/false is not a directory"* ]] + + # tarball doesn't exist (extension provided) run ch-tar2dir does_not_exist.tar.gz "$ch_imgdir" echo "$output" [[ $status -eq 1 ]] - [[ $output = *"can't read does_not_exist.tar.gz"* ]] + [[ $output = *"can't read: does_not_exist.tar.gz"* ]] + ! [[ $output = *"can't read: does_not_exist.tar.gz.tar.gz"* ]] + ! [[ $output = *"can't read: does_not_exist.tar.xz"* ]] + [[ $output = *"no input found" ]] + + # tarball doesn't exist (extension inferred, doesn't contain "tar") + run ch-tar2dir does_not_exist "$ch_imgdir" + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"can't read: does_not_exist"* ]] + [[ $output = *"can't read: does_not_exist.tar.gz"* ]] + [[ $output = *"can't read: does_not_exist.tar.xz"* ]] + [[ $output = *"no input found"* ]] + + # tarball doesn't exist (bad extension containing "tar") + run ch-tar2dir does_not_exist.tar.foo "$ch_imgdir" + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"can't read: does_not_exist.tar.foo"* ]] + ! [[ $output = *"can't read: does_not_exist.tar.foo.tar.gz"* ]] + ! [[ $output = *"can't read: does_not_exist.tar.foo.tar.xz"* ]] + [[ $output = *"no input found"* ]] # tarball exists but isn't readable touch "${BATS_TMPDIR}/unreadable.tar.gz" @@ -33,6 +79,19 @@ run ch-tar2dir "${BATS_TMPDIR}/unreadable.tar.gz" "$ch_imgdir" echo "$output" [[ $status -eq 1 ]] - [[ $output = *"can't read ${BATS_TMPDIR}/unreadable.tar.gz"* ]] + [[ $output = *"can't read: ${BATS_TMPDIR}/unreadable.tar.gz"* ]] + [[ $output = *"no input found"* ]] + + # file exists but has bad extension + touch "${BATS_TMPDIR}/foo.bar" + run ch-tar2dir "${BATS_TMPDIR}/foo.bar" "$ch_imgdir" + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"unknown extension: ${BATS_TMPDIR}/foo.bar"* ]] + touch "${BATS_TMPDIR}/foo.tar.bar" + run ch-tar2dir "${BATS_TMPDIR}/foo.tar.bar" "$ch_imgdir" + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"unknown extension: ${BATS_TMPDIR}/foo.tar.bar"* ]] } diff -Nru charliecloud-0.9.6/test/travis.sh charliecloud-0.9.10/test/travis.sh --- charliecloud-0.9.6/test/travis.sh 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/travis.sh 2019-05-14 20:34:32.000000000 +0000 @@ -26,6 +26,14 @@ tar xf charliecloud-*.tar.gz cd charliecloud-* ;; + export-bats) + (cd doc-src && make) + make export-bats + mv charliecloud-*.tar.gz "$PREFIX" + cd "$PREFIX" + tar xf charliecloud-*.tar.gz + cd charliecloud-* + ;; archive) # The Travis image already has Bats installed. git archive HEAD --prefix=charliecloud/ -o "$PREFIX/charliecloud.tar" @@ -35,34 +43,31 @@ ;; esac -if [[ $PKG_BUILD ]]; then - for i in packaging/*/travis.sh; do $i; done - # FIXME: If we continue with the rest of the tests after building the - # packages, they hang in "make test-all", I believe in test "ch-build - # python3" but I have not been able to verify this. - exit -fi - make bin/ch-run --version if [[ $INSTALL ]]; then sudo make install PREFIX="$PREFIX" - cd "$PREFIX/share/doc/charliecloud" + cd "$PREFIX/libexec/charliecloud" fi cd test make where-bats -make test +make test-build + +if [[ $SUDO_RM_AFTER_BUILD ]]; then + sudo rm /etc/sudoers.d/travis +fi +if [[ $SUDO_AVOID_AFTER_BUILD ]]; then + export CH_TEST_DONT_SUDO=yes +fi +if ( sudo -v ); then + echo "have sudo" +else + echo "don't have sudo" +fi +echo "\$CH_TEST_DONT_SUDO=$CH_TEST_DONT_SUDO" -# To test without Docker, move the binary out of the way. -DOCKER=$(command -v docker) -sudo mv "$DOCKER" "$DOCKER.tmp" - -make test - -# For Travis, this isn't really necessary, since the VM will go away -# immediately after this script exits. However, restore the binary to enable -# testing this script in other environments. -sudo mv "$DOCKER.tmp" "$DOCKER" +make test-run +make test-test diff -Nru charliecloud-0.9.6/test/travis.yml charliecloud-0.9.10/test/travis.yml --- charliecloud-0.9.6/test/travis.yml 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/test/travis.yml 2019-05-14 20:34:32.000000000 +0000 @@ -10,57 +10,72 @@ # it up in parallel. It would be nice if "make test-build" could be done # serially before splitting into parallel jobs. # -# TARBALL= # build in Git checkout & use embedded Bats -# TARBALL=archive # build from "git archive" tarball & use system Bats -# TARBALL=export # build from "make export" tarball & use embedded Bats -# INSTALL= # run from build directory -# INSTALL=yes # make install to /usr/local, run that one +# TARBALL= # build in Git checkout & use embedded Bats +# TARBALL=archive # build from "git archive" tarball & use system Bats +# TARBALL=export # build from "make export" tarball & use system Bats +# TARBALL=export-bats # build from "make export" tarball & use embedded Bats +# INSTALL= # run from build directory +# INSTALL=yes # make install to /usr/local, run that one # -# Package builds can also be tested. In this case, INSTALL is ignored, as is -# the main test suite. These tests are much faster. +# Additional options: # -# PKG_BUILD=yes # build (but don't test) distribution packages, then exit -# INSTALL_PV=yes # install pv ("pipe view") +# MINIMAL_DEPS # test with minimal dependencies (no fancy tools) +# NO_DOCKER # remove Docker before testing +# SUDO_RM_AFTER_BUILD # remove sudo privileges after build +# SUDO_AVOID_AFTER_BUILD # set CH_TEST_DONT_SUDO after build # env: -# Package building (fast). - - TARBALL= PKG_BUILD=yes - - TARBALL=archive PKG_BUILD=yes # Complete matrix of TARBALL and INSTALL. - - TARBALL= INSTALL= - - TARBALL= INSTALL=yes - - TARBALL=archive INSTALL= - - TARBALL=archive INSTALL=yes - - TARBALL=export INSTALL= - - TARBALL=export INSTALL=yes + - TARBALL= INSTALL= + - TARBALL= INSTALL=yes + - TARBALL=archive INSTALL= +# - TARBALL=archive INSTALL=yes + - TARBALL=export INSTALL= + - TARBALL=export INSTALL=yes + - TARBALL=export-bats INSTALL= +# - TARBALL=export-bats INSTALL=yes # Extra conditions - - TARBALL= INSTALL= INSTALL_PV=yes + - TARBALL= INSTALL= MINIMAL_DEPS=yes + - TARBALL= INSTALL= NO_DOCKER=yes + - TARBALL= INSTALL= SUDO_RM_AFTER_BUILD=yes + - TARBALL= INSTALL= SUDO_AVOID_AFTER_BUILD=yes # One full-scope test. This will finish last by a lot. # (Disabled because it gives a >10-minute gap in output, so Travis times out.) # - TARBALL= INSTALL= CH_TEST_SCOPE=full addons: apt: + sources: + - sourceline: 'ppa:projectatomic/ppa' packages: + - buildah + - cri-o-runc # runc from Project Atomic PPA, not Ubuntu - fakeroot - pigz + - pv - python3-pip + - skopeo install: - - if [ -n "$INSTALL_PV" ]; then sudo apt-get install pv; fi # We need Python 3 because Sphinx 1.8.0 doesn't work right under Python 2 (see # issue #241). Travis provides images pre-installed with Python 3, but it's in # a virtualenv and unavailable by default under sudo, in package builds, and # maybe elsewhere. It's simpler and fast enough to install it with apt-get. - - pip3 --version - - sudo pip3 install sphinx sphinx-rtd-theme -# ShellCheck: Use upstream's binary build because it's newer than what comes -# with Travis. - - sudo rm -f /usr/local/bin/shellcheck - - wget -nv https://shellcheck.storage.googleapis.com/shellcheck-v0.6.0.linux.x86_64.tar.xz - - tar xvf shellcheck-v0.6.0.linux.x86_64.tar.xz - - sudo mv shellcheck-v0.6.0/shellcheck /usr/local/bin - - shellcheck --version + - if [ -z "$MINIMAL_DEPS" ]; then + pip3 --version; + sudo pip3 install sphinx sphinx-rtd-theme; + fi +# umoci provides a binary build; no appropriate Ubuntu packages AFAICT. + - if [ -z "$MINIMAL_DEPS" ]; then + wget -nv https://github.com/openSUSE/umoci/releases/download/v0.4.3/umoci.amd64; + sudo chmod 755 umoci.amd64; + sudo mv umoci.amd64 /usr/local/bin/umoci; + umoci --version; + fi +# Tests should still pass with only the basics installed. + - if [ -n "$MINIMAL_DEPS" ]; then + sudo apt-get purge buildah cri-o-runc pv skopeo; + fi before_script: - getconf _NPROCESSORS_ONLN @@ -72,6 +87,11 @@ - export CH_TEST_PERMDIRS='/var/tmp /run' - unset JAVA_HOME # otherwise Spark tries to use host's Java - for d in $CH_TEST_PERMDIRS; do sudo test/make-perms-test $d $USER nobody; done + - sudo usermod --add-subuids 10000-65536 $USER + - sudo usermod --add-subgids 10000-65536 $USER + - if [ -n "$NO_DOCKER" ]; then + sudo rm $(command -v docker); + fi script: - test/travis.sh diff -Nru charliecloud-0.9.6/.travis.yml charliecloud-0.9.10/.travis.yml --- charliecloud-0.9.6/.travis.yml 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/.travis.yml 2019-05-14 20:34:32.000000000 +0000 @@ -10,57 +10,72 @@ # it up in parallel. It would be nice if "make test-build" could be done # serially before splitting into parallel jobs. # -# TARBALL= # build in Git checkout & use embedded Bats -# TARBALL=archive # build from "git archive" tarball & use system Bats -# TARBALL=export # build from "make export" tarball & use embedded Bats -# INSTALL= # run from build directory -# INSTALL=yes # make install to /usr/local, run that one +# TARBALL= # build in Git checkout & use embedded Bats +# TARBALL=archive # build from "git archive" tarball & use system Bats +# TARBALL=export # build from "make export" tarball & use system Bats +# TARBALL=export-bats # build from "make export" tarball & use embedded Bats +# INSTALL= # run from build directory +# INSTALL=yes # make install to /usr/local, run that one # -# Package builds can also be tested. In this case, INSTALL is ignored, as is -# the main test suite. These tests are much faster. +# Additional options: # -# PKG_BUILD=yes # build (but don't test) distribution packages, then exit -# INSTALL_PV=yes # install pv ("pipe view") +# MINIMAL_DEPS # test with minimal dependencies (no fancy tools) +# NO_DOCKER # remove Docker before testing +# SUDO_RM_AFTER_BUILD # remove sudo privileges after build +# SUDO_AVOID_AFTER_BUILD # set CH_TEST_DONT_SUDO after build # env: -# Package building (fast). - - TARBALL= PKG_BUILD=yes - - TARBALL=archive PKG_BUILD=yes # Complete matrix of TARBALL and INSTALL. - - TARBALL= INSTALL= - - TARBALL= INSTALL=yes - - TARBALL=archive INSTALL= - - TARBALL=archive INSTALL=yes - - TARBALL=export INSTALL= - - TARBALL=export INSTALL=yes + - TARBALL= INSTALL= + - TARBALL= INSTALL=yes + - TARBALL=archive INSTALL= +# - TARBALL=archive INSTALL=yes + - TARBALL=export INSTALL= + - TARBALL=export INSTALL=yes + - TARBALL=export-bats INSTALL= +# - TARBALL=export-bats INSTALL=yes # Extra conditions - - TARBALL= INSTALL= INSTALL_PV=yes + - TARBALL= INSTALL= MINIMAL_DEPS=yes + - TARBALL= INSTALL= NO_DOCKER=yes + - TARBALL= INSTALL= SUDO_RM_AFTER_BUILD=yes + - TARBALL= INSTALL= SUDO_AVOID_AFTER_BUILD=yes # One full-scope test. This will finish last by a lot. # (Disabled because it gives a >10-minute gap in output, so Travis times out.) # - TARBALL= INSTALL= CH_TEST_SCOPE=full addons: apt: + sources: + - sourceline: 'ppa:projectatomic/ppa' packages: + - buildah + - cri-o-runc # runc from Project Atomic PPA, not Ubuntu - fakeroot - pigz + - pv - python3-pip + - skopeo install: - - if [ -n "$INSTALL_PV" ]; then sudo apt-get install pv; fi # We need Python 3 because Sphinx 1.8.0 doesn't work right under Python 2 (see # issue #241). Travis provides images pre-installed with Python 3, but it's in # a virtualenv and unavailable by default under sudo, in package builds, and # maybe elsewhere. It's simpler and fast enough to install it with apt-get. - - pip3 --version - - sudo pip3 install sphinx sphinx-rtd-theme -# ShellCheck: Use upstream's binary build because it's newer than what comes -# with Travis. - - sudo rm -f /usr/local/bin/shellcheck - - wget -nv https://shellcheck.storage.googleapis.com/shellcheck-v0.6.0.linux.x86_64.tar.xz - - tar xvf shellcheck-v0.6.0.linux.x86_64.tar.xz - - sudo mv shellcheck-v0.6.0/shellcheck /usr/local/bin - - shellcheck --version + - if [ -z "$MINIMAL_DEPS" ]; then + pip3 --version; + sudo pip3 install sphinx sphinx-rtd-theme; + fi +# umoci provides a binary build; no appropriate Ubuntu packages AFAICT. + - if [ -z "$MINIMAL_DEPS" ]; then + wget -nv https://github.com/openSUSE/umoci/releases/download/v0.4.3/umoci.amd64; + sudo chmod 755 umoci.amd64; + sudo mv umoci.amd64 /usr/local/bin/umoci; + umoci --version; + fi +# Tests should still pass with only the basics installed. + - if [ -n "$MINIMAL_DEPS" ]; then + sudo apt-get purge buildah cri-o-runc pv skopeo; + fi before_script: - getconf _NPROCESSORS_ONLN @@ -72,6 +87,11 @@ - export CH_TEST_PERMDIRS='/var/tmp /run' - unset JAVA_HOME # otherwise Spark tries to use host's Java - for d in $CH_TEST_PERMDIRS; do sudo test/make-perms-test $d $USER nobody; done + - sudo usermod --add-subuids 10000-65536 $USER + - sudo usermod --add-subgids 10000-65536 $USER + - if [ -n "$NO_DOCKER" ]; then + sudo rm $(command -v docker); + fi script: - test/travis.sh diff -Nru charliecloud-0.9.6/VERSION charliecloud-0.9.10/VERSION --- charliecloud-0.9.6/VERSION 2018-12-13 20:49:21.000000000 +0000 +++ charliecloud-0.9.10/VERSION 2019-05-14 20:34:32.000000000 +0000 @@ -1 +1 @@ -0.9.6 +0.9.10